2024-12-10 00:13:57,363 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 00:13:57,377 main DEBUG Took 0.012204 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 00:13:57,378 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 00:13:57,378 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 00:13:57,379 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 00:13:57,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,391 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 00:13:57,405 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,408 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,409 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,411 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,412 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,414 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,415 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,417 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,418 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,419 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,420 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,421 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,422 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 00:13:57,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,423 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 00:13:57,426 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 00:13:57,428 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 00:13:57,431 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 00:13:57,432 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 00:13:57,434 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 00:13:57,434 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 00:13:57,447 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 00:13:57,450 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 00:13:57,453 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 00:13:57,453 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 00:13:57,454 main DEBUG createAppenders(={Console}) 2024-12-10 00:13:57,455 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-10 00:13:57,455 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 00:13:57,456 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-10 00:13:57,456 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 00:13:57,457 main DEBUG OutputStream closed 2024-12-10 00:13:57,457 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 00:13:57,458 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 00:13:57,458 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-10 00:13:57,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 00:13:57,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 00:13:57,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 00:13:57,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 00:13:57,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 00:13:57,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 00:13:57,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 00:13:57,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 00:13:57,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 00:13:57,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 00:13:57,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 00:13:57,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 00:13:57,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 00:13:57,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 00:13:57,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 00:13:57,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 00:13:57,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 00:13:57,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 00:13:57,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 00:13:57,573 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-10 00:13:57,574 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 00:13:57,575 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-10T00:13:57,878 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072 2024-12-10 00:13:57,882 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 00:13:57,883 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T00:13:57,895 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-10T00:13:57,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T00:13:57,944 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b, deleteOnExit=true 2024-12-10T00:13:57,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T00:13:57,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/test.cache.data in system properties and HBase conf 2024-12-10T00:13:57,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T00:13:57,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.log.dir in system properties and HBase conf 2024-12-10T00:13:57,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T00:13:57,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T00:13:57,950 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T00:13:58,079 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T00:13:58,209 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T00:13:58,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T00:13:58,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T00:13:58,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T00:13:58,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T00:13:58,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T00:13:58,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T00:13:58,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T00:13:58,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T00:13:58,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T00:13:58,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/nfs.dump.dir in system properties and HBase conf 2024-12-10T00:13:58,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/java.io.tmpdir in system properties and HBase conf 2024-12-10T00:13:58,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T00:13:58,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T00:13:58,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T00:13:59,714 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T00:13:59,793 INFO [Time-limited test {}] log.Log(170): Logging initialized @3193ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T00:13:59,869 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T00:13:59,958 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T00:13:59,985 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T00:13:59,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T00:13:59,987 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T00:14:00,003 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T00:14:00,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.log.dir/,AVAILABLE} 2024-12-10T00:14:00,008 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T00:14:00,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/java.io.tmpdir/jetty-localhost-45931-hadoop-hdfs-3_4_1-tests_jar-_-any-14886165220266900034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T00:14:00,245 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45931} 2024-12-10T00:14:00,245 INFO [Time-limited test {}] server.Server(415): Started @3646ms 2024-12-10T00:14:00,947 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T00:14:00,953 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T00:14:00,954 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T00:14:00,954 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T00:14:00,954 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T00:14:00,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.log.dir/,AVAILABLE} 2024-12-10T00:14:00,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T00:14:01,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/java.io.tmpdir/jetty-localhost-34693-hadoop-hdfs-3_4_1-tests_jar-_-any-652105642689397110/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T00:14:01,106 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34693} 2024-12-10T00:14:01,106 INFO [Time-limited test {}] server.Server(415): Started @4508ms 2024-12-10T00:14:01,170 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T00:14:03,624 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/data/data2/current/BP-85213971-172.17.0.2-1733789639131/current, will proceed with Du for space computation calculation, 2024-12-10T00:14:03,624 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/data/data1/current/BP-85213971-172.17.0.2-1733789639131/current, will proceed with Du for space computation calculation, 2024-12-10T00:14:03,673 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T00:14:03,739 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca22cd8737bd37b with lease ID 0x11da117653e7679a: Processing first storage report for DS-f7201ec0-4054-4520-8b26-bc2011faafef from datanode DatanodeRegistration(127.0.0.1:33323, datanodeUuid=ede03299-6f89-48ab-bc21-9d22c2a3de54, infoPort=38879, infoSecurePort=0, ipcPort=35771, storageInfo=lv=-57;cid=testClusterID;nsid=700807718;c=1733789639131) 2024-12-10T00:14:03,742 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca22cd8737bd37b with lease ID 0x11da117653e7679a: from storage DS-f7201ec0-4054-4520-8b26-bc2011faafef node DatanodeRegistration(127.0.0.1:33323, datanodeUuid=ede03299-6f89-48ab-bc21-9d22c2a3de54, infoPort=38879, infoSecurePort=0, ipcPort=35771, storageInfo=lv=-57;cid=testClusterID;nsid=700807718;c=1733789639131), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-10T00:14:03,756 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca22cd8737bd37b with lease ID 0x11da117653e7679a: Processing first storage report for DS-8e693f5d-4993-41ff-8529-bf3bf0397223 from datanode DatanodeRegistration(127.0.0.1:33323, datanodeUuid=ede03299-6f89-48ab-bc21-9d22c2a3de54, infoPort=38879, infoSecurePort=0, ipcPort=35771, storageInfo=lv=-57;cid=testClusterID;nsid=700807718;c=1733789639131) 2024-12-10T00:14:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca22cd8737bd37b with lease ID 0x11da117653e7679a: from storage DS-8e693f5d-4993-41ff-8529-bf3bf0397223 node DatanodeRegistration(127.0.0.1:33323, datanodeUuid=ede03299-6f89-48ab-bc21-9d22c2a3de54, infoPort=38879, infoSecurePort=0, ipcPort=35771, storageInfo=lv=-57;cid=testClusterID;nsid=700807718;c=1733789639131), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T00:14:03,799 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072 2024-12-10T00:14:03,894 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/zookeeper_0, clientPort=50532, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T00:14:03,907 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50532 2024-12-10T00:14:03,922 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:03,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:04,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741825_1001 (size=7) 2024-12-10T00:14:04,663 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 with version=8 2024-12-10T00:14:04,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/hbase-staging 2024-12-10T00:14:04,778 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T00:14:05,001 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a67c4886b4f7:0 server-side Connection retries=45 2024-12-10T00:14:05,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,016 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:14:05,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:14:05,152 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T00:14:05,226 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T00:14:05,239 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T00:14:05,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:14:05,269 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4609 (auto-detected) 2024-12-10T00:14:05,270 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T00:14:05,290 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35723 2024-12-10T00:14:05,317 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35723 connecting to ZooKeeper ensemble=127.0.0.1:50532 2024-12-10T00:14:05,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357230x0, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:14:05,475 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35723-0x1000d127b280000 connected 2024-12-10T00:14:05,649 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:05,652 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:05,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:14:05,672 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410, hbase.cluster.distributed=false 2024-12-10T00:14:05,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:14:05,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35723 2024-12-10T00:14:05,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35723 2024-12-10T00:14:05,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35723 2024-12-10T00:14:05,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35723 2024-12-10T00:14:05,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35723 2024-12-10T00:14:05,864 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a67c4886b4f7:0 server-side Connection retries=45 2024-12-10T00:14:05,866 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,866 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,867 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T00:14:05,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T00:14:05,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T00:14:05,871 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T00:14:05,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T00:14:05,876 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36911 2024-12-10T00:14:05,879 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36911 connecting to ZooKeeper ensemble=127.0.0.1:50532 2024-12-10T00:14:05,881 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:05,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:05,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369110x0, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T00:14:05,938 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369110x0, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:14:05,938 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36911-0x1000d127b280001 connected 2024-12-10T00:14:05,944 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T00:14:05,960 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T00:14:05,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T00:14:05,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T00:14:05,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36911 2024-12-10T00:14:05,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36911 2024-12-10T00:14:05,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36911 2024-12-10T00:14:05,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36911 2024-12-10T00:14:05,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36911 2024-12-10T00:14:06,014 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a67c4886b4f7:35723 2024-12-10T00:14:06,016 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:06,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:14:06,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:14:06,035 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:06,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T00:14:06,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:06,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:06,072 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T00:14:06,074 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a67c4886b4f7,35723,1733789644834 from backup master directory 2024-12-10T00:14:06,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:14:06,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:06,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T00:14:06,086 WARN [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:14:06,087 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:06,090 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T00:14:06,093 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T00:14:06,172 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/hbase.id] with ID: 0a958df3-9645-4748-962c-d558219e47f8 2024-12-10T00:14:06,172 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/.tmp/hbase.id 2024-12-10T00:14:06,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741826_1002 (size=42) 2024-12-10T00:14:06,599 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/.tmp/hbase.id]:[hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/hbase.id] 2024-12-10T00:14:06,677 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T00:14:06,684 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T00:14:06,717 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 31ms. 2024-12-10T00:14:06,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:06,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:06,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741827_1003 (size=196) 2024-12-10T00:14:07,243 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:14:07,246 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T00:14:07,262 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:07,267 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:14:07,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741828_1004 (size=1189) 2024-12-10T00:14:07,321 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store 2024-12-10T00:14:07,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741829_1005 (size=34) 2024-12-10T00:14:07,357 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T00:14:07,361 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:07,363 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:14:07,364 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:14:07,364 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:14:07,367 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:14:07,367 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:14:07,367 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:14:07,369 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733789647363Disabling compacts and flushes for region at 1733789647363Disabling writes for close at 1733789647367 (+4 ms)Writing region close event to WAL at 1733789647367Closed at 1733789647367 2024-12-10T00:14:07,372 WARN [master/a67c4886b4f7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/.initializing 2024-12-10T00:14:07,373 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/WALs/a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:07,386 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:14:07,407 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67c4886b4f7%2C35723%2C1733789644834, suffix=, logDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/WALs/a67c4886b4f7,35723,1733789644834, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/oldWALs, maxLogs=10 2024-12-10T00:14:07,446 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/WALs/a67c4886b4f7,35723,1733789644834/a67c4886b4f7%2C35723%2C1733789644834.1733789647420, exclude list is [], retry=0 2024-12-10T00:14:07,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33323,DS-f7201ec0-4054-4520-8b26-bc2011faafef,DISK] 2024-12-10T00:14:07,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T00:14:07,505 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/WALs/a67c4886b4f7,35723,1733789644834/a67c4886b4f7%2C35723%2C1733789644834.1733789647420 2024-12-10T00:14:07,506 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38879:38879)] 2024-12-10T00:14:07,507 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:14:07,508 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:07,511 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,513 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T00:14:07,584 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:07,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:07,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T00:14:07,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:07,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:07,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T00:14:07,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:07,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:07,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T00:14:07,599 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:07,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:07,601 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,604 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,606 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,611 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,612 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,617 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T00:14:07,622 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T00:14:07,627 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:14:07,629 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64154600, jitterRate=-0.04402196407318115}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T00:14:07,636 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733789647529Initializing all the Stores at 1733789647531 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789647532 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789647532Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789647533 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789647533Cleaning up temporary data from old regions at 1733789647613 (+80 ms)Region opened successfully at 1733789647635 (+22 ms) 2024-12-10T00:14:07,637 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T00:14:07,673 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e91f407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67c4886b4f7/172.17.0.2:0 2024-12-10T00:14:07,703 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T00:14:07,714 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T00:14:07,714 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T00:14:07,717 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T00:14:07,719 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T00:14:07,723 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-10T00:14:07,723 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T00:14:07,775 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T00:14:07,787 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T00:14:07,809 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T00:14:07,812 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T00:14:07,814 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T00:14:07,920 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T00:14:07,923 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T00:14:07,927 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T00:14:07,979 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T00:14:07,981 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T00:14:07,998 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T00:14:08,018 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T00:14:08,030 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T00:14:08,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:14:08,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T00:14:08,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,044 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a67c4886b4f7,35723,1733789644834, sessionid=0x1000d127b280000, setting cluster-up flag (Was=false) 2024-12-10T00:14:08,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,104 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T00:14:08,106 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:08,156 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T00:14:08,159 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:08,167 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T00:14:08,208 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(746): ClusterId : 0a958df3-9645-4748-962c-d558219e47f8 2024-12-10T00:14:08,212 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T00:14:08,224 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T00:14:08,224 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T00:14:08,232 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T00:14:08,233 DEBUG [RS:0;a67c4886b4f7:36911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322d57eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67c4886b4f7/172.17.0.2:0 2024-12-10T00:14:08,257 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a67c4886b4f7:36911 2024-12-10T00:14:08,262 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T00:14:08,262 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T00:14:08,262 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T00:14:08,266 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67c4886b4f7,35723,1733789644834 with port=36911, startcode=1733789645811 2024-12-10T00:14:08,280 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T00:14:08,283 DEBUG [RS:0;a67c4886b4f7:36911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T00:14:08,293 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T00:14:08,302 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T00:14:08,310 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a67c4886b4f7,35723,1733789644834 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T00:14:08,339 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a67c4886b4f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:14:08,346 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a67c4886b4f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:14:08,346 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a67c4886b4f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:14:08,346 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T00:14:08,347 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a67c4886b4f7:0, corePoolSize=5, maxPoolSize=5 2024-12-10T00:14:08,347 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a67c4886b4f7:0, corePoolSize=10, maxPoolSize=10 2024-12-10T00:14:08,347 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,349 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a67c4886b4f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:14:08,350 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,353 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733789678353 2024-12-10T00:14:08,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:14:08,357 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T00:14:08,357 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T00:14:08,358 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35723 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:08,359 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T00:14:08,364 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T00:14:08,364 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T00:14:08,365 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T00:14:08,365 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T00:14:08,367 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:08,368 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T00:14:08,383 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,387 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T00:14:08,388 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T00:14:08,389 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T00:14:08,397 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T00:14:08,397 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T00:14:08,400 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T00:14:08,400 WARN [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T00:14:08,400 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.large.0-1733789648399,5,FailOnTimeoutGroup] 2024-12-10T00:14:08,403 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.small.0-1733789648400,5,FailOnTimeoutGroup] 2024-12-10T00:14:08,403 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,403 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T00:14:08,405 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,405 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741831_1007 (size=1321) 2024-12-10T00:14:08,409 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T00:14:08,410 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:14:08,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741832_1008 (size=32) 2024-12-10T00:14:08,501 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67c4886b4f7,35723,1733789644834 with port=36911, startcode=1733789645811 2024-12-10T00:14:08,504 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35723 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,507 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35723 {}] master.ServerManager(517): Registering regionserver=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,518 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:14:08,518 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37473 2024-12-10T00:14:08,518 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T00:14:08,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:14:08,537 DEBUG [RS:0;a67c4886b4f7:36911 {}] zookeeper.ZKUtil(111): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,537 WARN [RS:0;a67c4886b4f7:36911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T00:14:08,539 INFO [RS:0;a67c4886b4f7:36911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:14:08,539 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a67c4886b4f7,36911,1733789645811] 2024-12-10T00:14:08,539 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,581 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T00:14:08,600 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T00:14:08,605 INFO [RS:0;a67c4886b4f7:36911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T00:14:08,605 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,612 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T00:14:08,619 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T00:14:08,621 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,621 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,622 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,622 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,622 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,622 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,623 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a67c4886b4f7:0, corePoolSize=2, maxPoolSize=2 2024-12-10T00:14:08,623 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,623 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,623 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,623 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,624 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,624 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a67c4886b4f7:0, corePoolSize=1, maxPoolSize=1 2024-12-10T00:14:08,624 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a67c4886b4f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:14:08,624 DEBUG [RS:0;a67c4886b4f7:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0, corePoolSize=3, maxPoolSize=3 2024-12-10T00:14:08,629 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,629 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,629 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,630 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,630 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,630 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,36911,1733789645811-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:14:08,658 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T00:14:08,661 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,36911,1733789645811-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,662 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,662 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.Replication(171): a67c4886b4f7,36911,1733789645811 started 2024-12-10T00:14:08,685 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:08,685 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1482): Serving as a67c4886b4f7,36911,1733789645811, RpcServer on a67c4886b4f7/172.17.0.2:36911, sessionid=0x1000d127b280001 2024-12-10T00:14:08,686 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T00:14:08,687 DEBUG [RS:0;a67c4886b4f7:36911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,687 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67c4886b4f7,36911,1733789645811' 2024-12-10T00:14:08,687 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T00:14:08,688 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T00:14:08,694 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T00:14:08,695 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T00:14:08,695 DEBUG [RS:0;a67c4886b4f7:36911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:08,695 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67c4886b4f7,36911,1733789645811' 2024-12-10T00:14:08,695 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T00:14:08,696 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T00:14:08,697 DEBUG [RS:0;a67c4886b4f7:36911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T00:14:08,697 INFO [RS:0;a67c4886b4f7:36911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T00:14:08,697 INFO [RS:0;a67c4886b4f7:36911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T00:14:08,803 INFO [RS:0;a67c4886b4f7:36911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T00:14:08,806 INFO [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67c4886b4f7%2C36911%2C1733789645811, suffix=, logDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/oldWALs, maxLogs=32 2024-12-10T00:14:08,827 DEBUG [RS:0;a67c4886b4f7:36911 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811/a67c4886b4f7%2C36911%2C1733789645811.1733789648809, exclude list is [], retry=0 2024-12-10T00:14:08,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33323,DS-f7201ec0-4054-4520-8b26-bc2011faafef,DISK] 2024-12-10T00:14:08,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:08,837 INFO [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811/a67c4886b4f7%2C36911%2C1733789645811.1733789648809 2024-12-10T00:14:08,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T00:14:08,837 DEBUG [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38879:38879)] 2024-12-10T00:14:08,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:14:08,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:08,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:08,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T00:14:08,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:14:08,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:08,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:08,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T00:14:08,848 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:14:08,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:08,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:08,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T00:14:08,852 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:14:08,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:08,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:08,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:14:08,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740 2024-12-10T00:14:08,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740 2024-12-10T00:14:08,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:14:08,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:14:08,861 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:14:08,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:14:08,867 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:14:08,868 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63951080, jitterRate=-0.04705464839935303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:14:08,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733789648834Initializing all the Stores at 1733789648836 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789648836Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789648837 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789648837Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789648837Cleaning up temporary data from old regions at 1733789648860 (+23 ms)Region opened successfully at 1733789648870 (+10 ms) 2024-12-10T00:14:08,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:14:08,870 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:14:08,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:14:08,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:14:08,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:14:08,872 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:14:08,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733789648870Disabling compacts and flushes for region at 1733789648870Disabling writes for close at 1733789648870Writing region close event to WAL at 1733789648871 (+1 ms)Closed at 1733789648872 (+1 ms) 2024-12-10T00:14:08,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:14:08,876 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T00:14:08,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T00:14:08,890 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:14:08,894 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T00:14:09,047 DEBUG [a67c4886b4f7:35723 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T00:14:09,059 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:09,066 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67c4886b4f7,36911,1733789645811, state=OPENING 2024-12-10T00:14:09,092 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T00:14:09,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:09,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:14:09,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:14:09,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:14:09,106 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T00:14:09,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:09,289 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T00:14:09,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T00:14:09,310 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T00:14:09,310 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T00:14:09,311 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T00:14:09,315 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67c4886b4f7%2C36911%2C1733789645811.meta, suffix=.meta, logDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/oldWALs, maxLogs=32 2024-12-10T00:14:09,338 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811/a67c4886b4f7%2C36911%2C1733789645811.meta.1733789649317.meta, exclude list is [], retry=0 2024-12-10T00:14:09,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33323,DS-f7201ec0-4054-4520-8b26-bc2011faafef,DISK] 2024-12-10T00:14:09,352 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/WALs/a67c4886b4f7,36911,1733789645811/a67c4886b4f7%2C36911%2C1733789645811.meta.1733789649317.meta 2024-12-10T00:14:09,353 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38879:38879)] 2024-12-10T00:14:09,354 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:14:09,356 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T00:14:09,360 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T00:14:09,365 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T00:14:09,371 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T00:14:09,372 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:09,373 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T00:14:09,373 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T00:14:09,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T00:14:09,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T00:14:09,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:09,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:09,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T00:14:09,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T00:14:09,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:09,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:09,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T00:14:09,388 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T00:14:09,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:09,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:09,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T00:14:09,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T00:14:09,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:09,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T00:14:09,394 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T00:14:09,396 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740 2024-12-10T00:14:09,400 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740 2024-12-10T00:14:09,403 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T00:14:09,403 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T00:14:09,405 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:14:09,409 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T00:14:09,411 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73032893, jitterRate=0.08827491104602814}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:14:09,411 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T00:14:09,415 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733789649374Writing region info on filesystem at 1733789649374Initializing all the Stores at 1733789649377 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789649377Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789649377Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789649377Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733789649377Cleaning up temporary data from old regions at 1733789649403 (+26 ms)Running coprocessor post-open hooks at 1733789649411 (+8 ms)Region opened successfully at 1733789649415 (+4 ms) 2024-12-10T00:14:09,426 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733789649278 2024-12-10T00:14:09,441 DEBUG [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T00:14:09,442 INFO [RS_OPEN_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T00:14:09,444 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:09,448 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67c4886b4f7,36911,1733789645811, state=OPEN 2024-12-10T00:14:09,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:14:09,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T00:14:09,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:14:09,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T00:14:09,568 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:09,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T00:14:09,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a67c4886b4f7,36911,1733789645811 in 461 msec 2024-12-10T00:14:09,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T00:14:09,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 696 msec 2024-12-10T00:14:09,588 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T00:14:09,589 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T00:14:09,611 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:09,612 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:09,632 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:09,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39221, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:09,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4510 sec 2024-12-10T00:14:09,670 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733789649670, completionTime=-1 2024-12-10T00:14:09,674 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T00:14:09,674 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T00:14:09,712 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-10T00:14:09,713 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733789709713 2024-12-10T00:14:09,713 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733789769713 2024-12-10T00:14:09,713 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-12-10T00:14:09,716 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,717 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,717 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,719 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a67c4886b4f7:35723, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,723 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,724 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,729 DEBUG [master/a67c4886b4f7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T00:14:09,773 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.685sec 2024-12-10T00:14:09,774 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T00:14:09,776 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T00:14:09,777 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T00:14:09,778 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T00:14:09,778 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T00:14:09,780 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T00:14:09,780 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T00:14:09,802 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T00:14:09,804 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T00:14:09,804 INFO [master/a67c4886b4f7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67c4886b4f7,35723,1733789644834-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T00:14:09,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfc638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:09,824 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T00:14:09,824 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T00:14:09,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:09,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:09,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:09,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a11d50b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:09,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:09,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:09,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc146ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:09,900 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:09,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:09,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:09,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=4840 2024-12-10T00:14:10,012 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:14:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e32371d 2024-12-10T00:14:10,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:14:10,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:14:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:14:10,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:14:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:10,066 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:14:10,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-10T00:14:10,069 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:10,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:14:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:10,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741835_1011 (size=1061) 2024-12-10T00:14:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:10,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:14:10,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741836_1012 (size=53) 2024-12-10T00:14:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:10,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:10,907 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a5f283ed44d3b86dba39726447c58806, disabling compactions & flushes 2024-12-10T00:14:10,907 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:10,907 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:10,907 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. after waiting 0 ms 2024-12-10T00:14:10,907 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:10,907 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:10,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5f283ed44d3b86dba39726447c58806: Waiting for close lock at 1733789650907Disabling compacts and flushes for region at 1733789650907Disabling writes for close at 1733789650907Writing region close event to WAL at 1733789650907Closed at 1733789650907 2024-12-10T00:14:10,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:14:10,918 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789650911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789650911"}]},"ts":"1733789650911"} 2024-12-10T00:14:10,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:14:10,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:14:10,932 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789650929"}]},"ts":"1733789650929"} 2024-12-10T00:14:10,938 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:14:10,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, ASSIGN}] 2024-12-10T00:14:10,943 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, ASSIGN 2024-12-10T00:14:10,945 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:14:11,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5f283ed44d3b86dba39726447c58806, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:11,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, ASSIGN because future has completed 2024-12-10T00:14:11,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:11,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:11,266 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:11,266 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:14:11,266 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,266 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:11,267 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,267 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,270 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,274 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:11,274 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5f283ed44d3b86dba39726447c58806 columnFamilyName A 2024-12-10T00:14:11,274 DEBUG [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:11,276 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(327): Store=a5f283ed44d3b86dba39726447c58806/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:11,276 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,279 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:11,279 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5f283ed44d3b86dba39726447c58806 columnFamilyName B 2024-12-10T00:14:11,279 DEBUG [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:11,280 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(327): Store=a5f283ed44d3b86dba39726447c58806/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:11,281 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,284 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:11,284 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5f283ed44d3b86dba39726447c58806 columnFamilyName C 2024-12-10T00:14:11,284 DEBUG [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:11,285 INFO [StoreOpener-a5f283ed44d3b86dba39726447c58806-1 {}] regionserver.HStore(327): Store=a5f283ed44d3b86dba39726447c58806/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:11,286 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:11,286 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,288 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,288 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,290 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,291 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,292 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:14:11,295 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,299 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:14:11,300 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a5f283ed44d3b86dba39726447c58806; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68624189, jitterRate=0.0225801020860672}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:14:11,301 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:11,301 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a5f283ed44d3b86dba39726447c58806: Running coprocessor pre-open hook at 1733789651267Writing region info on filesystem at 1733789651267Initializing all the Stores at 1733789651269 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789651269Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789651270 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789651270Cleaning up temporary data from old regions at 1733789651291 (+21 ms)Running coprocessor post-open hooks at 1733789651301 (+10 ms)Region opened successfully at 1733789651301 2024-12-10T00:14:11,304 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., pid=6, masterSystemTime=1733789651259 2024-12-10T00:14:11,307 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:11,307 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:11,309 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5f283ed44d3b86dba39726447c58806, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:11,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:14:11,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T00:14:11,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 in 212 msec 2024-12-10T00:14:11,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T00:14:11,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, ASSIGN in 380 msec 2024-12-10T00:14:11,328 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:14:11,328 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789651328"}]},"ts":"1733789651328"} 2024-12-10T00:14:11,331 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:14:11,333 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:14:11,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2820 sec 2024-12-10T00:14:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T00:14:12,232 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:12,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7943524a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,248 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:14:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-12-10T00:14:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:14:12,269 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:14:12,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,280 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,298 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,315 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167ce6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23efbfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,334 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c04dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2ac0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,345 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,351 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c912c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:12,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:12,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71258907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:12,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cfd059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:12,371 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,375 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,377 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,379 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,382 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,382 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,384 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,391 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:12,413 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-10T00:14:12,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:12,434 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:12,436 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,438 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:12,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:12,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:14:12,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:12,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:14:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:12,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:12,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/bebe78d2f80f4d03bac9e6de2a865ffa is 50, key is test_row_0/A:col10/1733789652433/Put/seqid=0 2024-12-10T00:14:12,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:12,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741837_1013 (size=12001) 2024-12-10T00:14:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789652582, max timestamp=1733789652656Num uniques 859; 2024-12-10T00:14:12,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:12,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789652582, max timestamp=1733789652654Num uniques 859; 2024-12-10T00:14:12,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:12,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789652582, max timestamp=1733789652654Num uniques 859; 2024-12-10T00:14:12,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:12,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789712774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789712793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789712801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789712802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:12,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789712804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:12,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789712922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789712922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789712931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789712933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789712932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:12,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/bebe78d2f80f4d03bac9e6de2a865ffa 2024-12-10T00:14:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789713141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789713142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789713142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789713152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789713151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:13,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:13,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/93de7997f98e43c39402b648b914bcab is 50, key is test_row_0/B:col10/1733789652433/Put/seqid=0 2024-12-10T00:14:13,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741838_1014 (size=12001) 2024-12-10T00:14:13,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/93de7997f98e43c39402b648b914bcab 2024-12-10T00:14:13,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/c3659f8bc7664b5985f331586f5a4037 is 50, key is test_row_0/C:col10/1733789652433/Put/seqid=0 2024-12-10T00:14:13,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:13,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741839_1015 (size=12001) 2024-12-10T00:14:13,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/c3659f8bc7664b5985f331586f5a4037 2024-12-10T00:14:13,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/bebe78d2f80f4d03bac9e6de2a865ffa as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa 2024-12-10T00:14:13,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T00:14:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T00:14:13,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/93de7997f98e43c39402b648b914bcab as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab 2024-12-10T00:14:13,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T00:14:13,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/c3659f8bc7664b5985f331586f5a4037 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037 2024-12-10T00:14:13,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T00:14:13,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a5f283ed44d3b86dba39726447c58806 in 1026ms, sequenceid=12, compaction requested=false 2024-12-10T00:14:13,500 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T00:14:13,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:13,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T00:14:13,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:13,522 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=134.18 KB heapSize=385.22 KB 2024-12-10T00:14:13,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789652768, max timestamp=1733789653512Num uniques 1018; 2024-12-10T00:14:13,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:13,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:13,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:13,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789652768, max timestamp=1733789653512Num uniques 1018; 2024-12-10T00:14:13,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:13,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789652768, max timestamp=1733789653512Num uniques 1018; 2024-12-10T00:14:13,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e569ce3b4641472589924ea003571b8e is 50, key is test_row_0/A:col10/1733789652768/Put/seqid=0 2024-12-10T00:14:13,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741840_1016 (size=12001) 2024-12-10T00:14:13,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789653688, max timestamp=1733789653760Num uniques 859; 2024-12-10T00:14:13,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789653688, max timestamp=1733789653764Num uniques 859; 2024-12-10T00:14:13,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:13,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789653688, max timestamp=1733789653759Num uniques 859; 2024-12-10T00:14:13,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:13,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789653837, max timestamp=1733789653895Num uniques 1018; 2024-12-10T00:14:13,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:14:13,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789653837, max timestamp=1733789653895Num uniques 1018; 2024-12-10T00:14:13,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:14:13,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789653837, max timestamp=1733789653886Num uniques 1018; 2024-12-10T00:14:13,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:13,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789653967, max timestamp=1733789653992Num uniques 1027; 2024-12-10T00:14:13,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T00:14:13,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789653967, max timestamp=1733789653992Num uniques 1077; 2024-12-10T00:14:13,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T00:14:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789653967, max timestamp=1733789653992Num uniques 1077; 2024-12-10T00:14:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T00:14:14,050 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e569ce3b4641472589924ea003571b8e 2024-12-10T00:14:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/0cade55df5214080af4aba8dc2dedc1c is 50, key is test_row_0/B:col10/1733789652768/Put/seqid=0 2024-12-10T00:14:14,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741841_1017 (size=12001) 2024-12-10T00:14:14,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789654063, max timestamp=1733789654108Num uniques 1036; 2024-12-10T00:14:14,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789654063, max timestamp=1733789654108Num uniques 1036; 2024-12-10T00:14:14,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-10T00:14:14,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-10T00:14:14,126 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/0cade55df5214080af4aba8dc2dedc1c 2024-12-10T00:14:14,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789654063, max timestamp=1733789654108Num uniques 1036; 2024-12-10T00:14:14,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-10T00:14:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ae02b6eb887649d8bf7db6000c94111e is 50, key is test_row_0/C:col10/1733789652768/Put/seqid=0 2024-12-10T00:14:14,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741842_1018 (size=12001) 2024-12-10T00:14:14,185 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ae02b6eb887649d8bf7db6000c94111e 2024-12-10T00:14:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e569ce3b4641472589924ea003571b8e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e 2024-12-10T00:14:14,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733789654167, max timestamp=1733789654198Num uniques 1009; 2024-12-10T00:14:14,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=18090, suffix heap size=132904,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=395 2024-12-10T00:14:14,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733789654167, max timestamp=1733789654198Num uniques 1009; 2024-12-10T00:14:14,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=18090, suffix heap size=132904,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=395 2024-12-10T00:14:14,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733789654167, max timestamp=1733789654198Num uniques 1009; 2024-12-10T00:14:14,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=18090, suffix heap size=132904,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=395 2024-12-10T00:14:14,221 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:14:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/0cade55df5214080af4aba8dc2dedc1c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c 2024-12-10T00:14:14,244 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:14:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ae02b6eb887649d8bf7db6000c94111e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e 2024-12-10T00:14:14,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:14:14,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=147.60 KB/151140 for a5f283ed44d3b86dba39726447c58806 in 747ms, sequenceid=50, compaction requested=false 2024-12-10T00:14:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-10T00:14:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-10T00:14:14,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=161.02 KB heapSize=442.55 KB 2024-12-10T00:14:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:14,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:14,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:14,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:14,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:14,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fa154560116b48b0b27e9c60a33378cb is 50, key is test_row_0/A:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:14,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T00:14:14,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-12-10T00:14:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789714293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9040 sec 2024-12-10T00:14:14,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789714317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789714320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789714326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789714337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741843_1019 (size=19321) 2024-12-10T00:14:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789714440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789714451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789714451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789714460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789714460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T00:14:14,590 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:14,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-10T00:14:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T00:14:14,595 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:14,597 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:14,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789714651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789714671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789714671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789714671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789714684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T00:14:14,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T00:14:14,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fa154560116b48b0b27e9c60a33378cb 2024-12-10T00:14:14,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:14,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4de3bb6047ee4007b57e666f07960e25 is 50, key is test_row_0/B:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:14,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741844_1020 (size=12151) 2024-12-10T00:14:14,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4de3bb6047ee4007b57e666f07960e25 2024-12-10T00:14:14,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ec092a05138643ada3e67d64af869999 is 50, key is test_row_0/C:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:14,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741845_1021 (size=12151) 2024-12-10T00:14:14,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ec092a05138643ada3e67d64af869999 2024-12-10T00:14:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fa154560116b48b0b27e9c60a33378cb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb 2024-12-10T00:14:14,896 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:14:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T00:14:14,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T00:14:14,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:14,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb, entries=300, sequenceid=155, filesize=18.9 K 2024-12-10T00:14:14,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4de3bb6047ee4007b57e666f07960e25 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25 2024-12-10T00:14:14,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T00:14:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ec092a05138643ada3e67d64af869999 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999 2024-12-10T00:14:14,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T00:14:14,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~442.50 KB/453120, currentSize=26.84 KB/27480 for a5f283ed44d3b86dba39726447c58806 in 689ms, sequenceid=155, compaction requested=true 2024-12-10T00:14:14,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:14,984 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:14,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:14,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:14,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:14,987 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:14,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:14,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:14,991 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:14,992 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:14,993 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:14,994 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:14,994 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,994 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:14,995 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=42.3 K 2024-12-10T00:14:14,995 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.3 K 2024-12-10T00:14:14,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 93de7997f98e43c39402b648b914bcab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789652433 2024-12-10T00:14:14,998 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cade55df5214080af4aba8dc2dedc1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789652768 2024-12-10T00:14:14,999 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de3bb6047ee4007b57e666f07960e25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:15,003 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebe78d2f80f4d03bac9e6de2a865ffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789652433 2024-12-10T00:14:15,006 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e569ce3b4641472589924ea003571b8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789652768 2024-12-10T00:14:15,007 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa154560116b48b0b27e9c60a33378cb, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:15,025 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T00:14:15,054 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#9 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:15,058 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/eb6710dcbe0d460b87dc3f097a22d6ad is 50, key is test_row_0/B:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:15,065 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:15,066 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/0d93aa02831b4a31a27e757ef07efb15 is 50, key is test_row_0/A:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:15,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T00:14:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:15,071 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T00:14:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:15,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741846_1022 (size=12254) 2024-12-10T00:14:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/7adf6fadf6d2403194e4bf19c166d0c6 is 50, key is test_row_0/A:col10/1733789654309/Put/seqid=0 2024-12-10T00:14:15,119 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/eb6710dcbe0d460b87dc3f097a22d6ad as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/eb6710dcbe0d460b87dc3f097a22d6ad 2024-12-10T00:14:15,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741847_1023 (size=12254) 2024-12-10T00:14:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741848_1024 (size=12151) 2024-12-10T00:14:15,154 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/7adf6fadf6d2403194e4bf19c166d0c6 2024-12-10T00:14:15,158 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into eb6710dcbe0d460b87dc3f097a22d6ad(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:15,161 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:15,170 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789654987; duration=0sec 2024-12-10T00:14:15,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:15,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:15,172 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:15,182 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36153 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:15,182 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:15,182 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:15,182 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.3 K 2024-12-10T00:14:15,184 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c3659f8bc7664b5985f331586f5a4037, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789652433 2024-12-10T00:14:15,185 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ae02b6eb887649d8bf7db6000c94111e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789652768 2024-12-10T00:14:15,187 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ec092a05138643ada3e67d64af869999, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:15,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/453f7b3bcbc343a1af621c8437a10e13 is 50, key is test_row_0/B:col10/1733789654309/Put/seqid=0 2024-12-10T00:14:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789655150, max timestamp=1733789655202Num uniques 859; 2024-12-10T00:14:15,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:15,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789655150, max timestamp=1733789655202Num uniques 859; 2024-12-10T00:14:15,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:15,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789655150, max timestamp=1733789655202Num uniques 859; 2024-12-10T00:14:15,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T00:14:15,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T00:14:15,222 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T00:14:15,224 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:14:15,224 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T00:14:15,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T00:14:15,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T00:14:15,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T00:14:15,226 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T00:14:15,252 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:15,253 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/6f5873262c7a4147930cd872f54762ec is 50, key is test_row_0/C:col10/1733789654259/Put/seqid=0 2024-12-10T00:14:15,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741849_1025 (size=12151) 2024-12-10T00:14:15,290 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/453f7b3bcbc343a1af621c8437a10e13 2024-12-10T00:14:15,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741850_1026 (size=12254) 2024-12-10T00:14:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/64319796ef5c412cbab574254a1fd6c4 is 50, key is test_row_0/C:col10/1733789654309/Put/seqid=0 2024-12-10T00:14:15,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789715317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789715324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789715336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789715347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789715336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741851_1027 (size=12151) 2024-12-10T00:14:15,391 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/64319796ef5c412cbab574254a1fd6c4 2024-12-10T00:14:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/7adf6fadf6d2403194e4bf19c166d0c6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6 2024-12-10T00:14:15,431 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T00:14:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/453f7b3bcbc343a1af621c8437a10e13 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13 2024-12-10T00:14:15,448 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T00:14:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/64319796ef5c412cbab574254a1fd6c4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4 2024-12-10T00:14:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789715470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789715470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,485 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T00:14:15,491 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a5f283ed44d3b86dba39726447c58806 in 420ms, sequenceid=165, compaction requested=false 2024-12-10T00:14:15,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:15,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:15,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-10T00:14:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-10T00:14:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:15,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T00:14:15,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:15,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:15,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:15,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:15,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:15,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:15,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fd1ed9c91bff476f8854c06da3cc7f3b is 50, key is test_row_0/A:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:15,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T00:14:15,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 902 msec 2024-12-10T00:14:15,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 922 msec 2024-12-10T00:14:15,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789715558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789715563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741852_1028 (size=14541) 2024-12-10T00:14:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789715572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,586 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/0d93aa02831b4a31a27e757ef07efb15 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/0d93aa02831b4a31a27e757ef07efb15 2024-12-10T00:14:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fd1ed9c91bff476f8854c06da3cc7f3b 2024-12-10T00:14:15,612 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 0d93aa02831b4a31a27e757ef07efb15(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:15,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:15,612 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789654963; duration=0sec 2024-12-10T00:14:15,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:15,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:15,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1eedd6ce81de455a8646244969b7d36c is 50, key is test_row_0/B:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:15,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741853_1029 (size=12151) 2024-12-10T00:14:15,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1eedd6ce81de455a8646244969b7d36c 2024-12-10T00:14:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/fc871e5e865f431da8b80ba8b6fc6f8c is 50, key is test_row_0/C:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:15,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789715690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789715690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789715679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789715691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789715700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T00:14:15,730 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:15,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:15,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741854_1030 (size=12151) 2024-12-10T00:14:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-10T00:14:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:15,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:15,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:15,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:15,744 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/6f5873262c7a4147930cd872f54762ec as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6f5873262c7a4147930cd872f54762ec 2024-12-10T00:14:15,766 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 6f5873262c7a4147930cd872f54762ec(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:15,766 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:15,766 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789654987; duration=0sec 2024-12-10T00:14:15,766 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:15,767 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:15,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T00:14:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:15,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:15,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789715910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789715910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789715921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789716010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789716019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T00:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:16,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/fc871e5e865f431da8b80ba8b6fc6f8c 2024-12-10T00:14:16,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fd1ed9c91bff476f8854c06da3cc7f3b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b 2024-12-10T00:14:16,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b, entries=200, sequenceid=204, filesize=14.2 K 2024-12-10T00:14:16,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1eedd6ce81de455a8646244969b7d36c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c 2024-12-10T00:14:16,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c, entries=150, sequenceid=204, filesize=11.9 K 2024-12-10T00:14:16,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/fc871e5e865f431da8b80ba8b6fc6f8c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c 2024-12-10T00:14:16,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c, entries=150, sequenceid=204, filesize=11.9 K 2024-12-10T00:14:16,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for a5f283ed44d3b86dba39726447c58806 in 714ms, sequenceid=204, compaction requested=true 2024-12-10T00:14:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:16,208 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:16,208 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:16,210 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:16,211 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:16,211 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,211 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/eb6710dcbe0d460b87dc3f097a22d6ad, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.7 K 2024-12-10T00:14:16,211 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38946 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:16,211 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:16,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T00:14:16,211 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,211 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/0d93aa02831b4a31a27e757ef07efb15, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=38.0 K 2024-12-10T00:14:16,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T00:14:16,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:16,213 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6710dcbe0d460b87dc3f097a22d6ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:16,213 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d93aa02831b4a31a27e757ef07efb15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:16,214 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adf6fadf6d2403194e4bf19c166d0c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733789654309 2024-12-10T00:14:16,214 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 453f7b3bcbc343a1af621c8437a10e13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733789654309 2024-12-10T00:14:16,214 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1ed9c91bff476f8854c06da3cc7f3b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:16,214 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eedd6ce81de455a8646244969b7d36c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fec0adc0da8a448c90bbc69b262edb15 is 50, key is test_row_0/A:col10/1733789655554/Put/seqid=0 2024-12-10T00:14:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:16,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:16,252 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:16,253 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fb5930abf9f5407e91526c39230ab241 is 50, key is test_row_0/A:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741855_1031 (size=12151) 2024-12-10T00:14:16,260 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fec0adc0da8a448c90bbc69b262edb15 2024-12-10T00:14:16,262 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:16,263 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/88431fcdb4cf4878adda012fa12b42e1 is 50, key is test_row_0/B:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/611b26482d494e9e95099e014a5b044a is 50, key is test_row_0/B:col10/1733789655554/Put/seqid=0 2024-12-10T00:14:16,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789656291, max timestamp=1733789656312Num uniques 859; 2024-12-10T00:14:16,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789656291, max timestamp=1733789656312Num uniques 859; 2024-12-10T00:14:16,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789656291, max timestamp=1733789656312Num uniques 859; 2024-12-10T00:14:16,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:16,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:16,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:16,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741856_1032 (size=12357) 2024-12-10T00:14:16,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741857_1033 (size=12357) 2024-12-10T00:14:16,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741858_1034 (size=12151) 2024-12-10T00:14:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:16,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/611b26482d494e9e95099e014a5b044a 2024-12-10T00:14:16,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789656380, max timestamp=1733789656402Num uniques 1018; 2024-12-10T00:14:16,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789656380, max timestamp=1733789656402Num uniques 1018; 2024-12-10T00:14:16,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:14:16,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:14:16,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789656380, max timestamp=1733789656402Num uniques 1018; 2024-12-10T00:14:16,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:14:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/90f8dd155da64741be79ee1fa4db8d59 is 50, key is test_row_0/C:col10/1733789655554/Put/seqid=0 2024-12-10T00:14:16,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741859_1035 (size=12151) 2024-12-10T00:14:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789716497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733789656487, max timestamp=1733789656514Num uniques 1077; 2024-12-10T00:14:16,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-10T00:14:16,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733789656487, max timestamp=1733789656514Num uniques 1077; 2024-12-10T00:14:16,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-10T00:14:16,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733789656487, max timestamp=1733789656514Num uniques 1077; 2024-12-10T00:14:16,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-10T00:14:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789716655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789716660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789716659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789716672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789716713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,738 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/88431fcdb4cf4878adda012fa12b42e1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/88431fcdb4cf4878adda012fa12b42e1 2024-12-10T00:14:16,762 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 88431fcdb4cf4878adda012fa12b42e1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:16,763 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:16,763 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789656208; duration=0sec 2024-12-10T00:14:16,763 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:16,763 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:16,764 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:16,766 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:16,767 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:16,768 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,769 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6f5873262c7a4147930cd872f54762ec, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.7 K 2024-12-10T00:14:16,770 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5873262c7a4147930cd872f54762ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733789654227 2024-12-10T00:14:16,771 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 64319796ef5c412cbab574254a1fd6c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733789654309 2024-12-10T00:14:16,772 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fb5930abf9f5407e91526c39230ab241 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fb5930abf9f5407e91526c39230ab241 2024-12-10T00:14:16,774 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fc871e5e865f431da8b80ba8b6fc6f8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:16,794 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into fb5930abf9f5407e91526c39230ab241(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:16,794 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:16,794 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789656208; duration=0sec 2024-12-10T00:14:16,794 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:16,794 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:16,807 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:16,808 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/100d5c83bdbc469696fdcd69f6317d45 is 50, key is test_row_0/C:col10/1733789655490/Put/seqid=0 2024-12-10T00:14:16,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789716810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789716810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789716810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789716810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789716832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741860_1036 (size=12357) 2024-12-10T00:14:16,866 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/100d5c83bdbc469696fdcd69f6317d45 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/100d5c83bdbc469696fdcd69f6317d45 2024-12-10T00:14:16,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/90f8dd155da64741be79ee1fa4db8d59 2024-12-10T00:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:16,890 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 100d5c83bdbc469696fdcd69f6317d45(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:16,890 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:16,890 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789656209; duration=0sec 2024-12-10T00:14:16,890 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:16,890 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:16,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fec0adc0da8a448c90bbc69b262edb15 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15 2024-12-10T00:14:16,908 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T00:14:16,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/611b26482d494e9e95099e014a5b044a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a 2024-12-10T00:14:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T00:14:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/90f8dd155da64741be79ee1fa4db8d59 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59 2024-12-10T00:14:16,948 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59, entries=150, sequenceid=215, filesize=11.9 K 2024-12-10T00:14:16,951 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a5f283ed44d3b86dba39726447c58806 in 739ms, sequenceid=215, compaction requested=false 2024-12-10T00:14:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:16,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-10T00:14:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-10T00:14:16,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-10T00:14:16,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-12-10T00:14:16,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2330 sec 2024-12-10T00:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:17,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-12-10T00:14:17,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789656617, max timestamp=1733789657023Num uniques 1086; 2024-12-10T00:14:17,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-10T00:14:17,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789656617, max timestamp=1733789657023Num uniques 1086; 2024-12-10T00:14:17,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-10T00:14:17,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789656617, max timestamp=1733789657023Num uniques 1086; 2024-12-10T00:14:17,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-10T00:14:17,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:17,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ad8f12b808bd454bafdc9e30b7b6280b is 50, key is test_row_0/A:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741861_1037 (size=12301) 2024-12-10T00:14:17,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ad8f12b808bd454bafdc9e30b7b6280b 2024-12-10T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789657097, max timestamp=1733789657116Num uniques 859; 2024-12-10T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789657097, max timestamp=1733789657116Num uniques 859; 2024-12-10T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:17,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789657097, max timestamp=1733789657116Num uniques 859; 2024-12-10T00:14:17,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:17,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/7599aff1437f4bb88124cdcb910a0cfe is 50, key is test_row_0/B:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741862_1038 (size=12301) 2024-12-10T00:14:17,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/7599aff1437f4bb88124cdcb910a0cfe 2024-12-10T00:14:17,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/f8e8fda4bdbf42f6924989f4927f5c2e is 50, key is test_row_0/C:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789657191, max timestamp=1733789657206Num uniques 1018; 2024-12-10T00:14:17,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:17,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789657191, max timestamp=1733789657206Num uniques 1018; 2024-12-10T00:14:17,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789657191, max timestamp=1733789657206Num uniques 1018; 2024-12-10T00:14:17,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:17,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:14:17,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741863_1039 (size=12301) 2024-12-10T00:14:17,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733789657268, max timestamp=1733789657322Num uniques 1027; 2024-12-10T00:14:17,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-10T00:14:17,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789657275, max timestamp=1733789657322Num uniques 1027; 2024-12-10T00:14:17,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-12-10T00:14:17,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733789657268, max timestamp=1733789657322Num uniques 1027; 2024-12-10T00:14:17,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733789657407, max timestamp=1733789657430Num uniques 1136; 2024-12-10T00:14:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733789657407, max timestamp=1733789657430Num uniques 1136; 2024-12-10T00:14:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789717424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733789657407, max timestamp=1733789657430Num uniques 1009; 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-10T00:14:17,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-10T00:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733789657473, max timestamp=1733789657492Num uniques 1095; 2024-12-10T00:14:17,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-10T00:14:17,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789657481, max timestamp=1733789657492Num uniques 1095; 2024-12-10T00:14:17,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-12-10T00:14:17,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789657481, max timestamp=1733789657492Num uniques 1095; 2024-12-10T00:14:17,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-12-10T00:14:17,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733789657526, max timestamp=1733789657572Num uniques 1009; 2024-12-10T00:14:17,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=18500, suffix heap size=132904,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=404 2024-12-10T00:14:17,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733789657526, max timestamp=1733789657572Num uniques 1104; 2024-12-10T00:14:17,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=18500, suffix heap size=142784,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=404 2024-12-10T00:14:17,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733789657526, max timestamp=1733789657572Num uniques 1009; 2024-12-10T00:14:17,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=18500, suffix heap size=132904,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=404 2024-12-10T00:14:17,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789717608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789717608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789717611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789717615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789717615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:17,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/f8e8fda4bdbf42f6924989f4927f5c2e 2024-12-10T00:14:17,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ad8f12b808bd454bafdc9e30b7b6280b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b 2024-12-10T00:14:17,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T00:14:17,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/7599aff1437f4bb88124cdcb910a0cfe as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe 2024-12-10T00:14:17,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T00:14:17,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/f8e8fda4bdbf42f6924989f4927f5c2e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e 2024-12-10T00:14:17,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e, entries=150, sequenceid=291, filesize=12.0 K 2024-12-10T00:14:17,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~83.58 KB/85584, currentSize=167.72 KB/171750 for a5f283ed44d3b86dba39726447c58806 in 693ms, sequenceid=291, compaction requested=true 2024-12-10T00:14:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:17,723 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:17,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:17,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:17,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:17,724 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:17,725 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:17,725 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:17,726 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:17,726 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fb5930abf9f5407e91526c39230ab241, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.9 K 2024-12-10T00:14:17,727 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:17,727 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:17,727 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:17,727 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/88431fcdb4cf4878adda012fa12b42e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.9 K 2024-12-10T00:14:17,727 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5930abf9f5407e91526c39230ab241, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:17,728 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 88431fcdb4cf4878adda012fa12b42e1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:17,728 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec0adc0da8a448c90bbc69b262edb15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733789655543 2024-12-10T00:14:17,729 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 611b26482d494e9e95099e014a5b044a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733789655543 2024-12-10T00:14:17,729 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad8f12b808bd454bafdc9e30b7b6280b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:17,730 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7599aff1437f4bb88124cdcb910a0cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:17,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733789657605, max timestamp=1733789657732Num uniques 1263; 2024-12-10T00:14:17,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=57838, new segment data size=7458, suffix heap size=159320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1263, new segment cells count=163 2024-12-10T00:14:17,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733789657605, max timestamp=1733789657732Num uniques 1263; 2024-12-10T00:14:17,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=57838, new segment data size=7458, suffix heap size=159320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1263, new segment cells count=163 2024-12-10T00:14:17,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=76.04 KB heapSize=204.66 KB 2024-12-10T00:14:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:17,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733789657605, max timestamp=1733789657732Num uniques 1263; 2024-12-10T00:14:17,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57838, new segment data size=7458, suffix heap size=159320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1263, new segment cells count=163 2024-12-10T00:14:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:17,750 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:17,751 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/aa9b2c4521a740d7bf5a2168c0d54ad7 is 50, key is test_row_0/A:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,761 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:17,762 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4d4b9e70e362407291e23319c8cdca72 is 50, key is test_row_0/B:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/05ae9a1ae3324af38513be2d85f9c183 is 50, key is test_row_0/A:col10/1733789657610/Put/seqid=0 2024-12-10T00:14:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741865_1041 (size=12609) 2024-12-10T00:14:17,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741864_1040 (size=12609) 2024-12-10T00:14:17,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741866_1042 (size=14741) 2024-12-10T00:14:17,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/05ae9a1ae3324af38513be2d85f9c183 2024-12-10T00:14:17,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789657761, max timestamp=1733789657798Num uniques 859; 2024-12-10T00:14:17,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:14:17,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789657761, max timestamp=1733789657800Num uniques 859; 2024-12-10T00:14:17,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T00:14:17,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789657761, max timestamp=1733789657800Num uniques 859; 2024-12-10T00:14:17,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T00:14:17,808 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/aa9b2c4521a740d7bf5a2168c0d54ad7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/aa9b2c4521a740d7bf5a2168c0d54ad7 2024-12-10T00:14:17,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/fea5c11de74748d6a33709b37280aa45 is 50, key is test_row_0/B:col10/1733789657610/Put/seqid=0 2024-12-10T00:14:17,823 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into aa9b2c4521a740d7bf5a2168c0d54ad7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:17,824 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:17,824 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789657723; duration=0sec 2024-12-10T00:14:17,824 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:17,824 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:17,824 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:17,828 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36809 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:17,829 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:17,829 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:17,829 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/100d5c83bdbc469696fdcd69f6317d45, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=35.9 K 2024-12-10T00:14:17,831 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100d5c83bdbc469696fdcd69f6317d45, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733789655286 2024-12-10T00:14:17,831 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4d4b9e70e362407291e23319c8cdca72 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4d4b9e70e362407291e23319c8cdca72 2024-12-10T00:14:17,833 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f8dd155da64741be79ee1fa4db8d59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733789655543 2024-12-10T00:14:17,834 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e8fda4bdbf42f6924989f4927f5c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:17,850 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 4d4b9e70e362407291e23319c8cdca72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:17,850 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:17,850 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789657723; duration=0sec 2024-12-10T00:14:17,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741867_1043 (size=12301) 2024-12-10T00:14:17,850 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:17,850 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:17,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733789657838, max timestamp=1733789657858Num uniques 1068; 2024-12-10T00:14:17,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-10T00:14:17,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733789657838, max timestamp=1733789657858Num uniques 1009; 2024-12-10T00:14:17,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-12-10T00:14:17,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733789657838, max timestamp=1733789657858Num uniques 1009; 2024-12-10T00:14:17,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=12268, suffix heap size=132904,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=268 2024-12-10T00:14:17,871 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:17,872 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/575064fda45c463e9ad4d4cb086c052a is 50, key is test_row_0/C:col10/1733789656690/Put/seqid=0 2024-12-10T00:14:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T00:14:17,890 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:17,891 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-10T00:14:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:17,901 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:17,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:17,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741868_1044 (size=12609) 2024-12-10T00:14:17,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789657885, max timestamp=1733789657921Num uniques 1127; 2024-12-10T00:14:17,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-10T00:14:17,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789657885, max timestamp=1733789657921Num uniques 1127; 2024-12-10T00:14:17,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-10T00:14:17,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789657885, max timestamp=1733789657921Num uniques 1127; 2024-12-10T00:14:17,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733789657975, max timestamp=1733789657995Num uniques 1036; 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733789657975, max timestamp=1733789657995Num uniques 1036; 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733789657975, max timestamp=1733789657995Num uniques 1036; 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-10T00:14:17,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-10T00:14:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:18,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733789658025, max timestamp=1733789658042Num uniques 1195; 2024-12-10T00:14:18,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-10T00:14:18,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733789658025, max timestamp=1733789658042Num uniques 1195; 2024-12-10T00:14:18,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-10T00:14:18,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733789658025, max timestamp=1733789658042Num uniques 1195; 2024-12-10T00:14:18,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=15800, suffix heap size=152248,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=345 2024-12-10T00:14:18,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789718177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789718178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789718179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789718181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789718200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:18,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/fea5c11de74748d6a33709b37280aa45 2024-12-10T00:14:18,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a901694f67eb4a94a6f1d310e1a8eab9 is 50, key is test_row_0/C:col10/1733789657610/Put/seqid=0 2024-12-10T00:14:18,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789718300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789718300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789718300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789718302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741869_1045 (size=12301) 2024-12-10T00:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a901694f67eb4a94a6f1d310e1a8eab9 2024-12-10T00:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789718310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/05ae9a1ae3324af38513be2d85f9c183 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183 2024-12-10T00:14:18,348 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/575064fda45c463e9ad4d4cb086c052a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/575064fda45c463e9ad4d4cb086c052a 2024-12-10T00:14:18,365 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 575064fda45c463e9ad4d4cb086c052a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:18,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:18,366 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789657723; duration=0sec 2024-12-10T00:14:18,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:18,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:18,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183, entries=200, sequenceid=416, filesize=14.4 K 2024-12-10T00:14:18,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/fea5c11de74748d6a33709b37280aa45 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45 2024-12-10T00:14:18,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45, entries=150, sequenceid=416, filesize=12.0 K 2024-12-10T00:14:18,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a901694f67eb4a94a6f1d310e1a8eab9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9 2024-12-10T00:14:18,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9, entries=150, sequenceid=416, filesize=12.0 K 2024-12-10T00:14:18,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.73 KB/87792, currentSize=154.31 KB/158010 for a5f283ed44d3b86dba39726447c58806 in 672ms, sequenceid=416, compaction requested=false 2024-12-10T00:14:18,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:18,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-12-10T00:14:18,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733789658127, max timestamp=1733789658513Num uniques 1204; 2024-12-10T00:14:18,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733789658127, max timestamp=1733789658513Num uniques 1204; 2024-12-10T00:14:18,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-12-10T00:14:18,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-12-10T00:14:18,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:18,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:18,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:18,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:18,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:18,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:18,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5b7b2ee70c0f4994bf663c142e1450d2 is 50, key is test_row_0/A:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741870_1046 (size=12301) 2024-12-10T00:14:18,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5b7b2ee70c0f4994bf663c142e1450d2 2024-12-10T00:14:18,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789718560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789718561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789718563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789718569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789718574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/5c0a67ec7c064632a25084f76ad73e8e is 50, key is test_row_0/B:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741871_1047 (size=12301) 2024-12-10T00:14:18,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/5c0a67ec7c064632a25084f76ad73e8e 2024-12-10T00:14:18,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/6b719866c3194dbdab93d2e3f0f04289 is 50, key is test_row_0/C:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:18,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741872_1048 (size=12301) 2024-12-10T00:14:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789718692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789718692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789718692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789718692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789718692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:18,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789718899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789718900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789718900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789718910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789718910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:18,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:19,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:19,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/6b719866c3194dbdab93d2e3f0f04289 2024-12-10T00:14:19,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5b7b2ee70c0f4994bf663c142e1450d2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2 2024-12-10T00:14:19,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2, entries=150, sequenceid=526, filesize=12.0 K 2024-12-10T00:14:19,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/5c0a67ec7c064632a25084f76ad73e8e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e 2024-12-10T00:14:19,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e, entries=150, sequenceid=526, filesize=12.0 K 2024-12-10T00:14:19,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/6b719866c3194dbdab93d2e3f0f04289 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289 2024-12-10T00:14:19,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289, entries=150, sequenceid=526, filesize=12.0 K 2024-12-10T00:14:19,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~210.53 KB/215584, currentSize=114.05 KB/116790 for a5f283ed44d3b86dba39726447c58806 in 631ms, sequenceid=526, compaction requested=true 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:19,148 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:19,148 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:19,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:19,149 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:19,150 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:19,150 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:19,150 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:19,150 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,150 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,150 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4d4b9e70e362407291e23319c8cdca72, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.3 K 2024-12-10T00:14:19,150 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/aa9b2c4521a740d7bf5a2168c0d54ad7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=38.7 K 2024-12-10T00:14:19,151 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9b2c4521a740d7bf5a2168c0d54ad7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:19,151 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ae9a1ae3324af38513be2d85f9c183, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789657605 2024-12-10T00:14:19,152 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d4b9e70e362407291e23319c8cdca72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:19,152 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fea5c11de74748d6a33709b37280aa45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789657605 2024-12-10T00:14:19,152 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7b2ee70c0f4994bf663c142e1450d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:19,153 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0a67ec7c064632a25084f76ad73e8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:19,178 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:19,179 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/f507560040d747c68259f1dffcac9353 is 50, key is test_row_0/B:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:19,185 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:19,186 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/399448e048dd473380bf9517fe18a3b4 is 50, key is test_row_0/A:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:19,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T00:14:19,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,191 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e53ef60dbb6146d18fc815dd86224585 is 50, key is test_row_0/A:col10/1733789658561/Put/seqid=0 2024-12-10T00:14:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:19,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:19,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741873_1049 (size=12711) 2024-12-10T00:14:19,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741874_1050 (size=12711) 2024-12-10T00:14:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741875_1051 (size=12301) 2024-12-10T00:14:19,234 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/399448e048dd473380bf9517fe18a3b4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/399448e048dd473380bf9517fe18a3b4 2024-12-10T00:14:19,246 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 399448e048dd473380bf9517fe18a3b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:19,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,247 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:19,247 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789659148; duration=0sec 2024-12-10T00:14:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789719239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789719241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,249 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:19,249 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:19,249 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:19,250 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:19,250 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:19,250 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:19,251 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/575064fda45c463e9ad4d4cb086c052a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.3 K 2024-12-10T00:14:19,251 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575064fda45c463e9ad4d4cb086c052a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733789656617 2024-12-10T00:14:19,252 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a901694f67eb4a94a6f1d310e1a8eab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789657605 2024-12-10T00:14:19,252 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b719866c3194dbdab93d2e3f0f04289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789719244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789719247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789719247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,282 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:19,283 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/9436527a4cb5486e9680b21689f2b6cf is 50, key is test_row_0/C:col10/1733789658167/Put/seqid=0 2024-12-10T00:14:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741876_1052 (size=12711) 2024-12-10T00:14:19,330 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/9436527a4cb5486e9680b21689f2b6cf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/9436527a4cb5486e9680b21689f2b6cf 2024-12-10T00:14:19,342 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 9436527a4cb5486e9680b21689f2b6cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:19,342 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:19,342 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789659148; duration=0sec 2024-12-10T00:14:19,342 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:19,343 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789719360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789719360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789719373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789719369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789719377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789719570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789719571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789719580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789719590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789719590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,634 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e53ef60dbb6146d18fc815dd86224585 2024-12-10T00:14:19,652 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/f507560040d747c68259f1dffcac9353 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/f507560040d747c68259f1dffcac9353 2024-12-10T00:14:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/528d96d7cbca44f186509fd50329fb49 is 50, key is test_row_0/B:col10/1733789658561/Put/seqid=0 2024-12-10T00:14:19,666 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into f507560040d747c68259f1dffcac9353(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:19,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:19,666 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789659148; duration=0sec 2024-12-10T00:14:19,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:19,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:19,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741877_1053 (size=12301) 2024-12-10T00:14:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789719881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789719881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789719890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789719899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789719899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:19,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:20,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/528d96d7cbca44f186509fd50329fb49 2024-12-10T00:14:20,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/8900b1d013f34d7291fda20a42083416 is 50, key is test_row_0/C:col10/1733789658561/Put/seqid=0 2024-12-10T00:14:20,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741878_1054 (size=12301) 2024-12-10T00:14:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789720390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789720399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789720410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789720420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:20,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789720420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:20,559 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/8900b1d013f34d7291fda20a42083416 2024-12-10T00:14:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/e53ef60dbb6146d18fc815dd86224585 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585 2024-12-10T00:14:20,583 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585, entries=150, sequenceid=546, filesize=12.0 K 2024-12-10T00:14:20,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/528d96d7cbca44f186509fd50329fb49 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49 2024-12-10T00:14:20,598 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49, entries=150, sequenceid=546, filesize=12.0 K 2024-12-10T00:14:20,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/8900b1d013f34d7291fda20a42083416 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416 2024-12-10T00:14:20,619 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416, entries=150, sequenceid=546, filesize=12.0 K 2024-12-10T00:14:20,622 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a5f283ed44d3b86dba39726447c58806 in 1430ms, sequenceid=546, compaction requested=false 2024-12-10T00:14:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-10T00:14:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-10T00:14:20,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T00:14:20,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7180 sec 2024-12-10T00:14:20,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.7420 sec 2024-12-10T00:14:21,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:14:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:21,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:21,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:21,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:21,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:21,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:21,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:21,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/c80ac8542d2e4b04a797fb4f1f0ff9ce is 50, key is test_row_0/A:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741879_1055 (size=14741) 2024-12-10T00:14:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789721479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789721479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789721491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789721495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789721498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789721600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789721600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789721610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789721610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789721610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789721809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789721810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789721820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789721829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789721830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:21,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/c80ac8542d2e4b04a797fb4f1f0ff9ce 2024-12-10T00:14:21,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/bc5f67cb4ab345498075bcef43f81fc7 is 50, key is test_row_0/B:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:21,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741880_1056 (size=12301) 2024-12-10T00:14:21,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/bc5f67cb4ab345498075bcef43f81fc7 2024-12-10T00:14:21,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/84413c26a52c4549a6ca70aa9d818933 is 50, key is test_row_0/C:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:21,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741881_1057 (size=12301) 2024-12-10T00:14:21,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/84413c26a52c4549a6ca70aa9d818933 2024-12-10T00:14:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/c80ac8542d2e4b04a797fb4f1f0ff9ce as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce 2024-12-10T00:14:22,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce, entries=200, sequenceid=564, filesize=14.4 K 2024-12-10T00:14:22,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/bc5f67cb4ab345498075bcef43f81fc7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7 2024-12-10T00:14:22,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7, entries=150, sequenceid=564, filesize=12.0 K 2024-12-10T00:14:22,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/84413c26a52c4549a6ca70aa9d818933 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933 2024-12-10T00:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T00:14:22,054 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:22,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-10T00:14:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:22,059 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:22,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:22,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:22,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933, entries=150, sequenceid=564, filesize=12.0 K 2024-12-10T00:14:22,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for a5f283ed44d3b86dba39726447c58806 in 656ms, sequenceid=564, compaction requested=true 2024-12-10T00:14:22,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:22,070 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:22,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:22,070 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:22,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:22,072 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:22,072 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:22,072 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:22,072 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:22,072 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,072 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,072 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/f507560040d747c68259f1dffcac9353, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.4 K 2024-12-10T00:14:22,072 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/399448e048dd473380bf9517fe18a3b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=38.8 K 2024-12-10T00:14:22,073 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f507560040d747c68259f1dffcac9353, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:22,073 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 399448e048dd473380bf9517fe18a3b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:22,073 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 528d96d7cbca44f186509fd50329fb49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733789658558 2024-12-10T00:14:22,075 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5f67cb4ab345498075bcef43f81fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659242 2024-12-10T00:14:22,076 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53ef60dbb6146d18fc815dd86224585, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733789658558 2024-12-10T00:14:22,077 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80ac8542d2e4b04a797fb4f1f0ff9ce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659228 2024-12-10T00:14:22,110 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:22,111 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2414ca2b554d4f608d7da3f83800b30d is 50, key is test_row_0/A:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:22,121 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#46 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:22,122 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/23a620edf7ac47d89762c0c98db6bbc3 is 50, key is test_row_0/B:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:22,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:22,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789661455, max timestamp=1733789662134Num uniques 859; 2024-12-10T00:14:22,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789661455, max timestamp=1733789662134Num uniques 859; 2024-12-10T00:14:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:22,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:22,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:22,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741882_1058 (size=12813) 2024-12-10T00:14:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741883_1059 (size=12813) 2024-12-10T00:14:22,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2bd929a529c9495c8134cd36a66a9817 is 50, key is test_row_0/A:col10/1733789661489/Put/seqid=0 2024-12-10T00:14:22,159 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2414ca2b554d4f608d7da3f83800b30d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2414ca2b554d4f608d7da3f83800b30d 2024-12-10T00:14:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:22,169 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/23a620edf7ac47d89762c0c98db6bbc3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/23a620edf7ac47d89762c0c98db6bbc3 2024-12-10T00:14:22,181 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 2414ca2b554d4f608d7da3f83800b30d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,181 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789662070; duration=0sec 2024-12-10T00:14:22,181 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 23a620edf7ac47d89762c0c98db6bbc3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,181 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789662070; duration=0sec 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:22,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:22,183 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:22,183 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:22,183 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,183 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/9436527a4cb5486e9680b21689f2b6cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.4 K 2024-12-10T00:14:22,183 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9436527a4cb5486e9680b21689f2b6cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733789658167 2024-12-10T00:14:22,184 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8900b1d013f34d7291fda20a42083416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1733789658558 2024-12-10T00:14:22,184 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84413c26a52c4549a6ca70aa9d818933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659242 2024-12-10T00:14:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741884_1060 (size=14741) 2024-12-10T00:14:22,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2bd929a529c9495c8134cd36a66a9817 2024-12-10T00:14:22,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1530ced4cfdd46a1951719739bb7d7e5 is 50, key is test_row_0/B:col10/1733789661489/Put/seqid=0 2024-12-10T00:14:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,218 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:22,219 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/b67896019cd24dd59eb3e44634fc7fb4 is 50, key is test_row_0/C:col10/1733789661411/Put/seqid=0 2024-12-10T00:14:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789722209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789662171, max timestamp=1733789662217Num uniques 859; 2024-12-10T00:14:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789722211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:22,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789662171, max timestamp=1733789662216Num uniques 859; 2024-12-10T00:14:22,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789662171, max timestamp=1733789662216Num uniques 859; 2024-12-10T00:14:22,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:22,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741885_1061 (size=12813) 2024-12-10T00:14:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741886_1062 (size=12301) 2024-12-10T00:14:22,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1530ced4cfdd46a1951719739bb7d7e5 2024-12-10T00:14:22,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/181c9c3e19a44510bc11cdf0105a96eb is 50, key is test_row_0/C:col10/1733789661489/Put/seqid=0 2024-12-10T00:14:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741887_1063 (size=12301) 2024-12-10T00:14:22,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/181c9c3e19a44510bc11cdf0105a96eb 2024-12-10T00:14:22,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2bd929a529c9495c8134cd36a66a9817 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817 2024-12-10T00:14:22,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817, entries=200, sequenceid=585, filesize=14.4 K 2024-12-10T00:14:22,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/1530ced4cfdd46a1951719739bb7d7e5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5 2024-12-10T00:14:22,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5, entries=150, sequenceid=585, filesize=12.0 K 2024-12-10T00:14:22,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/181c9c3e19a44510bc11cdf0105a96eb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb 2024-12-10T00:14:22,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789722315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb, entries=150, sequenceid=585, filesize=12.0 K 2024-12-10T00:14:22,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for a5f283ed44d3b86dba39726447c58806 in 208ms, sequenceid=585, compaction requested=false 2024-12-10T00:14:22,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:22,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T00:14:22,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733789662251, max timestamp=1733789662346Num uniques 1018; 2024-12-10T00:14:22,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-10T00:14:22,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:14:22,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/1e11adaaddb74521878abd0f1f780bb1 is 50, key is test_row_0/A:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:22,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:22,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741888_1064 (size=14741) 2024-12-10T00:14:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:22,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/1e11adaaddb74521878abd0f1f780bb1 2024-12-10T00:14:22,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789722386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789722387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789722389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789722389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 is 50, key is test_row_0/B:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:22,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741889_1065 (size=12301) 2024-12-10T00:14:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789722451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789722500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789722510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789722510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789722511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,656 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/b67896019cd24dd59eb3e44634fc7fb4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b67896019cd24dd59eb3e44634fc7fb4 2024-12-10T00:14:22,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789722660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,670 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into b67896019cd24dd59eb3e44634fc7fb4(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:22,670 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,670 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789662071; duration=0sec 2024-12-10T00:14:22,671 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:22,671 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:22,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789722721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789722721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789722723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789722725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:22,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:22,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 2024-12-10T00:14:22,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/5ec208f5a4fa4db0bde66d70e193b394 is 50, key is test_row_0/C:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:22,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741890_1066 (size=12301) 2024-12-10T00:14:22,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/5ec208f5a4fa4db0bde66d70e193b394 2024-12-10T00:14:22,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/1e11adaaddb74521878abd0f1f780bb1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1 2024-12-10T00:14:22,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1, entries=200, sequenceid=625, filesize=14.4 K 2024-12-10T00:14:22,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 2024-12-10T00:14:22,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1, entries=150, sequenceid=625, filesize=12.0 K 2024-12-10T00:14:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/5ec208f5a4fa4db0bde66d70e193b394 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394 2024-12-10T00:14:22,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394, entries=150, sequenceid=625, filesize=12.0 K 2024-12-10T00:14:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789722970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:22,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~307.39 KB/314768, currentSize=73.80 KB/75570 for a5f283ed44d3b86dba39726447c58806 in 633ms, sequenceid=625, compaction requested=true 2024-12-10T00:14:22,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:22,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:22,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:22,980 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:22,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:22,981 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:22,982 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:22,982 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:22,982 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,982 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:22,982 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2414ca2b554d4f608d7da3f83800b30d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=41.3 K 2024-12-10T00:14:22,983 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:22,983 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,983 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/23a620edf7ac47d89762c0c98db6bbc3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.5 K 2024-12-10T00:14:22,984 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a620edf7ac47d89762c0c98db6bbc3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659242 2024-12-10T00:14:22,984 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1530ced4cfdd46a1951719739bb7d7e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1733789661455 2024-12-10T00:14:22,985 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e5aafb50b8d49d9aebcd0a95ec06ae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662338 2024-12-10T00:14:22,991 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2414ca2b554d4f608d7da3f83800b30d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659242 2024-12-10T00:14:22,992 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd929a529c9495c8134cd36a66a9817, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1733789661455 2024-12-10T00:14:22,992 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e11adaaddb74521878abd0f1f780bb1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662251 2024-12-10T00:14:22,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T00:14:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:22,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:14:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:22,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:22,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:23,000 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:23,001 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4ea69405161f4b24b886634cd6d0d737 is 50, key is test_row_0/B:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:23,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5361022ccca84e3094792bc69d3acf20 is 50, key is test_row_0/A:col10/1733789662376/Put/seqid=0 2024-12-10T00:14:23,019 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:23,019 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fbfbb4cc3e98438593c17d182ee8548c is 50, key is test_row_0/A:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:23,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741891_1067 (size=12915) 2024-12-10T00:14:23,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741892_1068 (size=12301) 2024-12-10T00:14:23,047 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5361022ccca84e3094792bc69d3acf20 2024-12-10T00:14:23,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741893_1069 (size=12915) 2024-12-10T00:14:23,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:23,056 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4ea69405161f4b24b886634cd6d0d737 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4ea69405161f4b24b886634cd6d0d737 2024-12-10T00:14:23,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/54f85ef23b0c4842b390c014a003eb6d is 50, key is test_row_0/B:col10/1733789662376/Put/seqid=0 2024-12-10T00:14:23,070 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/fbfbb4cc3e98438593c17d182ee8548c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fbfbb4cc3e98438593c17d182ee8548c 2024-12-10T00:14:23,072 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 4ea69405161f4b24b886634cd6d0d737(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:23,072 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:23,072 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789662980; duration=0sec 2024-12-10T00:14:23,072 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:23,073 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:23,073 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:23,081 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:23,081 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:23,081 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:23,082 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b67896019cd24dd59eb3e44634fc7fb4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.5 K 2024-12-10T00:14:23,082 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b67896019cd24dd59eb3e44634fc7fb4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1733789659242 2024-12-10T00:14:23,083 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 181c9c3e19a44510bc11cdf0105a96eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1733789661455 2024-12-10T00:14:23,086 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into fbfbb4cc3e98438593c17d182ee8548c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:23,086 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:23,086 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789662980; duration=0sec 2024-12-10T00:14:23,086 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:23,087 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:23,087 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec208f5a4fa4db0bde66d70e193b394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662338 2024-12-10T00:14:23,107 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:23,109 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/b6290186fa7547348e6be84dd0b06e90 is 50, key is test_row_0/C:col10/1733789662338/Put/seqid=0 2024-12-10T00:14:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741894_1070 (size=12301) 2024-12-10T00:14:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789723092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789723098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789723108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,114 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/54f85ef23b0c4842b390c014a003eb6d 2024-12-10T00:14:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789723110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3eb5022af6874fafbbad5a9c33aa1c2e is 50, key is test_row_0/C:col10/1733789662376/Put/seqid=0 2024-12-10T00:14:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741895_1071 (size=12915) 2024-12-10T00:14:23,182 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/b6290186fa7547348e6be84dd0b06e90 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b6290186fa7547348e6be84dd0b06e90 2024-12-10T00:14:23,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741896_1072 (size=12301) 2024-12-10T00:14:23,184 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3eb5022af6874fafbbad5a9c33aa1c2e 2024-12-10T00:14:23,193 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into b6290186fa7547348e6be84dd0b06e90(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:23,193 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:23,193 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789662981; duration=0sec 2024-12-10T00:14:23,193 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:23,193 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:23,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/5361022ccca84e3094792bc69d3acf20 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20 2024-12-10T00:14:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:23,206 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20, entries=150, sequenceid=640, filesize=12.0 K 2024-12-10T00:14:23,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/54f85ef23b0c4842b390c014a003eb6d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d 2024-12-10T00:14:23,219 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d, entries=150, sequenceid=640, filesize=12.0 K 2024-12-10T00:14:23,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3eb5022af6874fafbbad5a9c33aa1c2e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e 2024-12-10T00:14:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789723220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789723220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789723220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789723230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,238 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e, entries=150, sequenceid=640, filesize=12.0 K 2024-12-10T00:14:23,239 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a5f283ed44d3b86dba39726447c58806 in 242ms, sequenceid=640, compaction requested=false 2024-12-10T00:14:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:23,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-10T00:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-10T00:14:23,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-10T00:14:23,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1820 sec 2024-12-10T00:14:23,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1910 sec 2024-12-10T00:14:23,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:23,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:23,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/9533b8fb7f984dcc96f078394fc2c20e is 50, key is test_row_0/A:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789723478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789723479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789723478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789723481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741897_1073 (size=12301) 2024-12-10T00:14:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789723493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789723600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789723600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789723610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789723613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789723809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789723810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789723820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789723822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:23,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/9533b8fb7f984dcc96f078394fc2c20e 2024-12-10T00:14:23,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/8f8ff537abb8427cb4ffa76453507f4c is 50, key is test_row_0/B:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:23,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741898_1074 (size=12301) 2024-12-10T00:14:24,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/8f8ff537abb8427cb4ffa76453507f4c 2024-12-10T00:14:24,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a60b614bb4d94a77b196896d20e7ebcd is 50, key is test_row_0/C:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:24,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741899_1075 (size=12301) 2024-12-10T00:14:24,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a60b614bb4d94a77b196896d20e7ebcd 2024-12-10T00:14:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789724130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789724130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789724140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789724140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/9533b8fb7f984dcc96f078394fc2c20e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e 2024-12-10T00:14:24,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e, entries=150, sequenceid=662, filesize=12.0 K 2024-12-10T00:14:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/8f8ff537abb8427cb4ffa76453507f4c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c 2024-12-10T00:14:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T00:14:24,218 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:24,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:24,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c, entries=150, sequenceid=662, filesize=12.0 K 2024-12-10T00:14:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-10T00:14:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/a60b614bb4d94a77b196896d20e7ebcd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd 2024-12-10T00:14:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T00:14:24,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:24,242 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:24,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:24,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd, entries=150, sequenceid=662, filesize=12.0 K 2024-12-10T00:14:24,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a5f283ed44d3b86dba39726447c58806 in 822ms, sequenceid=662, compaction requested=true 2024-12-10T00:14:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:24,267 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:24,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:24,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:24,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:24,268 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:24,277 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:24,278 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:24,278 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:24,278 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4ea69405161f4b24b886634cd6d0d737, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.6 K 2024-12-10T00:14:24,279 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea69405161f4b24b886634cd6d0d737, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662338 2024-12-10T00:14:24,279 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f85ef23b0c4842b390c014a003eb6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733789662369 2024-12-10T00:14:24,280 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f8ff537abb8427cb4ffa76453507f4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:24,285 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:24,285 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:24,285 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:24,285 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fbfbb4cc3e98438593c17d182ee8548c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.6 K 2024-12-10T00:14:24,289 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfbb4cc3e98438593c17d182ee8548c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662338 2024-12-10T00:14:24,297 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5361022ccca84e3094792bc69d3acf20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733789662369 2024-12-10T00:14:24,299 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9533b8fb7f984dcc96f078394fc2c20e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:24,299 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:24,300 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/d18363fe9d38407292a12a9081dfe7c2 is 50, key is test_row_0/B:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:24,348 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:24,349 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d77cc21ec1284c11ad45361db24ac8ba is 50, key is test_row_0/A:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T00:14:24,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741900_1076 (size=13017) 2024-12-10T00:14:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741901_1077 (size=13017) 2024-12-10T00:14:24,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T00:14:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:24,398 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:24,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:24,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a577d51186fe4f6ab2acc245bb9c0f28 is 50, key is test_row_0/A:col10/1733789663477/Put/seqid=0 2024-12-10T00:14:24,437 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/d18363fe9d38407292a12a9081dfe7c2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d18363fe9d38407292a12a9081dfe7c2 2024-12-10T00:14:24,473 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d77cc21ec1284c11ad45361db24ac8ba as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d77cc21ec1284c11ad45361db24ac8ba 2024-12-10T00:14:24,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741902_1078 (size=12301) 2024-12-10T00:14:24,488 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=675 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a577d51186fe4f6ab2acc245bb9c0f28 2024-12-10T00:14:24,497 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into d18363fe9d38407292a12a9081dfe7c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:24,498 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:24,498 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789664267; duration=0sec 2024-12-10T00:14:24,498 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:24,498 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:24,498 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:24,509 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:24,509 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:24,509 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:24,510 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b6290186fa7547348e6be84dd0b06e90, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.6 K 2024-12-10T00:14:24,511 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6290186fa7547348e6be84dd0b06e90, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=625, earliestPutTs=1733789662338 2024-12-10T00:14:24,512 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb5022af6874fafbbad5a9c33aa1c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733789662369 2024-12-10T00:14:24,517 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a60b614bb4d94a77b196896d20e7ebcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:24,517 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into d77cc21ec1284c11ad45361db24ac8ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:24,517 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:24,517 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789664267; duration=0sec 2024-12-10T00:14:24,518 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:24,518 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:24,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e8adaf00cc794900b93785974ebb14d8 is 50, key is test_row_0/B:col10/1733789663477/Put/seqid=0 2024-12-10T00:14:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T00:14:24,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741903_1079 (size=12301) 2024-12-10T00:14:24,579 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:24,580 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/29c0dbcaeaaa4cce880660fe01d1b643 is 50, key is test_row_0/C:col10/1733789663089/Put/seqid=0 2024-12-10T00:14:24,580 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=675 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e8adaf00cc794900b93785974ebb14d8 2024-12-10T00:14:24,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/918b5bd4c0e44f268ff03936f5661de3 is 50, key is test_row_0/C:col10/1733789663477/Put/seqid=0 2024-12-10T00:14:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741904_1080 (size=13017) 2024-12-10T00:14:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741905_1081 (size=12301) 2024-12-10T00:14:24,655 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/29c0dbcaeaaa4cce880660fe01d1b643 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29c0dbcaeaaa4cce880660fe01d1b643 2024-12-10T00:14:24,657 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=675 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/918b5bd4c0e44f268ff03936f5661de3 2024-12-10T00:14:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789724654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789724660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a577d51186fe4f6ab2acc245bb9c0f28 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28 2024-12-10T00:14:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789724660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789724697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28, entries=150, sequenceid=675, filesize=12.0 K 2024-12-10T00:14:24,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e8adaf00cc794900b93785974ebb14d8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8 2024-12-10T00:14:24,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789724703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,713 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 29c0dbcaeaaa4cce880660fe01d1b643(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:24,714 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:24,714 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789664268; duration=0sec 2024-12-10T00:14:24,714 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:24,714 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:24,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8, entries=150, sequenceid=675, filesize=12.0 K 2024-12-10T00:14:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/918b5bd4c0e44f268ff03936f5661de3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3 2024-12-10T00:14:24,725 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3, entries=150, sequenceid=675, filesize=12.0 K 2024-12-10T00:14:24,726 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a5f283ed44d3b86dba39726447c58806 in 328ms, sequenceid=675, compaction requested=false 2024-12-10T00:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-10T00:14:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-10T00:14:24,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-10T00:14:24,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 485 msec 2024-12-10T00:14:24,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 507 msec 2024-12-10T00:14:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789664651, max timestamp=1733789664801Num uniques 859; 2024-12-10T00:14:24,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:14:24,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:24,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d7542e5ba7fa4124be21c418d748e5f7 is 50, key is test_row_0/A:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:24,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789724839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789724841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789724845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741906_1082 (size=14741) 2024-12-10T00:14:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T00:14:24,870 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:24,871 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-10T00:14:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:24,873 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:24,874 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:24,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789724950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789724950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789724959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:25,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789725160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789725169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789725170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:25,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d7542e5ba7fa4124be21c418d748e5f7 2024-12-10T00:14:25,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/36b275fe595d4f26b8b6a3aff46901bd is 50, key is test_row_0/B:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741907_1083 (size=12301) 2024-12-10T00:14:25,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789725469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789725481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789725481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/36b275fe595d4f26b8b6a3aff46901bd 2024-12-10T00:14:25,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ac422a221cc44b96a14dce5f51d536c4 is 50, key is test_row_0/C:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:25,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741908_1084 (size=12301) 2024-12-10T00:14:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789725720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789725720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,727 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,730 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:25,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:25,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:25,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:25,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789725980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789725999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:26,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:26,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789726000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:26,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:26,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ac422a221cc44b96a14dce5f51d536c4 2024-12-10T00:14:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d7542e5ba7fa4124be21c418d748e5f7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7 2024-12-10T00:14:26,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7, entries=200, sequenceid=699, filesize=14.4 K 2024-12-10T00:14:26,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:26,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/36b275fe595d4f26b8b6a3aff46901bd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd 2024-12-10T00:14:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:26,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd, entries=150, sequenceid=699, filesize=12.0 K 2024-12-10T00:14:26,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ac422a221cc44b96a14dce5f51d536c4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4 2024-12-10T00:14:26,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4, entries=150, sequenceid=699, filesize=12.0 K 2024-12-10T00:14:26,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for a5f283ed44d3b86dba39726447c58806 in 1351ms, sequenceid=699, compaction requested=true 2024-12-10T00:14:26,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:26,156 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:26,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:26,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:26,158 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:26,158 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:26,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:26,158 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,158 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d77cc21ec1284c11ad45361db24ac8ba, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=39.1 K 2024-12-10T00:14:26,159 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77cc21ec1284c11ad45361db24ac8ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:26,159 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a577d51186fe4f6ab2acc245bb9c0f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=675, earliestPutTs=1733789663475 2024-12-10T00:14:26,160 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7542e5ba7fa4124be21c418d748e5f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664651 2024-12-10T00:14:26,161 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:26,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:26,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:26,171 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,171 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d18363fe9d38407292a12a9081dfe7c2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.7 K 2024-12-10T00:14:26,172 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d18363fe9d38407292a12a9081dfe7c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:26,173 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e8adaf00cc794900b93785974ebb14d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=675, earliestPutTs=1733789663475 2024-12-10T00:14:26,174 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b275fe595d4f26b8b6a3aff46901bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664693 2024-12-10T00:14:26,189 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:26,190 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/98889acfec404eedaa9e6241af7daebf is 50, key is test_row_0/A:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:26,192 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:26,192 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/880da7b92d624b42b4fc4cd391b6c5a2 is 50, key is test_row_0/B:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741910_1086 (size=13119) 2024-12-10T00:14:26,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741909_1085 (size=13119) 2024-12-10T00:14:26,228 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/880da7b92d624b42b4fc4cd391b6c5a2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/880da7b92d624b42b4fc4cd391b6c5a2 2024-12-10T00:14:26,229 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/98889acfec404eedaa9e6241af7daebf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/98889acfec404eedaa9e6241af7daebf 2024-12-10T00:14:26,239 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 98889acfec404eedaa9e6241af7daebf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:26,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:26,239 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789666156; duration=0sec 2024-12-10T00:14:26,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:26,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:26,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:26,242 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 880da7b92d624b42b4fc4cd391b6c5a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:26,242 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:26,242 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789666158; duration=0sec 2024-12-10T00:14:26,242 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:26,242 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:26,244 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:26,244 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:26,244 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,244 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29c0dbcaeaaa4cce880660fe01d1b643, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.7 K 2024-12-10T00:14:26,245 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c0dbcaeaaa4cce880660fe01d1b643, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1733789663089 2024-12-10T00:14:26,245 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918b5bd4c0e44f268ff03936f5661de3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=675, earliestPutTs=1733789663475 2024-12-10T00:14:26,246 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac422a221cc44b96a14dce5f51d536c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664693 2024-12-10T00:14:26,263 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#74 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:26,264 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/aae437500a5e43fda716c8c0e288b25a is 50, key is test_row_0/C:col10/1733789664695/Put/seqid=0 2024-12-10T00:14:26,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T00:14:26,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T00:14:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:26,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:26,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741911_1087 (size=13119) 2024-12-10T00:14:26,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a6248bab15a542978e842863bd2230fc is 50, key is test_row_0/A:col10/1733789664842/Put/seqid=0 2024-12-10T00:14:26,319 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/aae437500a5e43fda716c8c0e288b25a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/aae437500a5e43fda716c8c0e288b25a 2024-12-10T00:14:26,328 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into aae437500a5e43fda716c8c0e288b25a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:26,328 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:26,328 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789666158; duration=0sec 2024-12-10T00:14:26,328 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:26,328 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:26,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741912_1088 (size=12301) 2024-12-10T00:14:26,389 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a6248bab15a542978e842863bd2230fc 2024-12-10T00:14:26,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e62b15f442814524b0a6bbe6e7b0b71a is 50, key is test_row_0/B:col10/1733789664842/Put/seqid=0 2024-12-10T00:14:26,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741913_1089 (size=12301) 2024-12-10T00:14:26,521 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e62b15f442814524b0a6bbe6e7b0b71a 2024-12-10T00:14:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/29e1a8292837476ba35bcae1a9e43d89 is 50, key is test_row_0/C:col10/1733789664842/Put/seqid=0 2024-12-10T00:14:26,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741914_1090 (size=12301) 2024-12-10T00:14:26,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=717 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/29e1a8292837476ba35bcae1a9e43d89 2024-12-10T00:14:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/a6248bab15a542978e842863bd2230fc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc 2024-12-10T00:14:26,734 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc, entries=150, sequenceid=717, filesize=12.0 K 2024-12-10T00:14:26,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/e62b15f442814524b0a6bbe6e7b0b71a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a 2024-12-10T00:14:26,748 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a, entries=150, sequenceid=717, filesize=12.0 K 2024-12-10T00:14:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/29e1a8292837476ba35bcae1a9e43d89 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89 2024-12-10T00:14:26,764 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89, entries=150, sequenceid=717, filesize=12.0 K 2024-12-10T00:14:26,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for a5f283ed44d3b86dba39726447c58806 in 480ms, sequenceid=717, compaction requested=false 2024-12-10T00:14:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-10T00:14:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-10T00:14:26,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-10T00:14:26,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-12-10T00:14:26,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.9340 sec 2024-12-10T00:14:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T00:14:27,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:27,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-10T00:14:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:27,029 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:27,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:27,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:27,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:27,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:27,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4ab279c0eeb6457a8df4ae0502019537 is 50, key is test_row_0/A:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741915_1091 (size=14741) 2024-12-10T00:14:27,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4ab279c0eeb6457a8df4ae0502019537 2024-12-10T00:14:27,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733789667069, max timestamp=1733789667119Num uniques 859; 2024-12-10T00:14:27,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T00:14:27,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/3afb8971ae9a4009946f22cde35f71ff is 50, key is test_row_0/B:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789727138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789727140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789727140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741916_1092 (size=12301) 2024-12-10T00:14:27,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T00:14:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789727253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789727253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789727253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T00:14:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789727460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789727460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789727461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T00:14:27,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:27,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:27,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/3afb8971ae9a4009946f22cde35f71ff 2024-12-10T00:14:27,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ef1b9f9c80084a13ad38a49bce049932 is 50, key is test_row_0/C:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741917_1093 (size=12301) 2024-12-10T00:14:27,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=728 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ef1b9f9c80084a13ad38a49bce049932 2024-12-10T00:14:27,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4ab279c0eeb6457a8df4ae0502019537 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537 2024-12-10T00:14:27,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537, entries=200, sequenceid=728, filesize=14.4 K 2024-12-10T00:14:27,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/3afb8971ae9a4009946f22cde35f71ff as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff 2024-12-10T00:14:27,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff, entries=150, sequenceid=728, filesize=12.0 K 2024-12-10T00:14:27,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/ef1b9f9c80084a13ad38a49bce049932 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932 2024-12-10T00:14:27,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932, entries=150, sequenceid=728, filesize=12.0 K 2024-12-10T00:14:27,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for a5f283ed44d3b86dba39726447c58806 in 587ms, sequenceid=728, compaction requested=true 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:27,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T00:14:27,623 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:27,624 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:27,625 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:27,625 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:27,625 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,625 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/98889acfec404eedaa9e6241af7daebf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=39.2 K 2024-12-10T00:14:27,626 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 98889acfec404eedaa9e6241af7daebf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664693 2024-12-10T00:14:27,626 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:27,626 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:27,626 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,626 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/aae437500a5e43fda716c8c0e288b25a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.8 K 2024-12-10T00:14:27,626 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a6248bab15a542978e842863bd2230fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733789664834 2024-12-10T00:14:27,627 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae437500a5e43fda716c8c0e288b25a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664693 2024-12-10T00:14:27,629 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab279c0eeb6457a8df4ae0502019537, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:27,629 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e1a8292837476ba35bcae1a9e43d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733789664834 2024-12-10T00:14:27,631 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1b9f9c80084a13ad38a49bce049932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:27,656 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#81 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:27,657 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:27,657 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/499208a82c894e28bf2fa541c059ec78 is 50, key is test_row_0/A:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,658 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/0999969158df4798b13eb40b7e85a600 is 50, key is test_row_0/C:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:27,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T00:14:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,682 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=138.65 KB heapSize=401.95 KB 2024-12-10T00:14:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/95305558b81c4fe49f9b9141f703ef8c is 50, key is test_row_0/A:col10/1733789667123/Put/seqid=0 2024-12-10T00:14:27,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741919_1095 (size=13221) 2024-12-10T00:14:27,733 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/0999969158df4798b13eb40b7e85a600 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0999969158df4798b13eb40b7e85a600 2024-12-10T00:14:27,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741918_1094 (size=13221) 2024-12-10T00:14:27,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:27,760 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 0999969158df4798b13eb40b7e85a600(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:27,760 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:27,760 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789667622; duration=0sec 2024-12-10T00:14:27,760 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:27,760 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:27,760 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741920_1096 (size=12301) 2024-12-10T00:14:27,763 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:27,763 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:27,763 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:27,763 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/880da7b92d624b42b4fc4cd391b6c5a2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.8 K 2024-12-10T00:14:27,769 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880da7b92d624b42b4fc4cd391b6c5a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733789664693 2024-12-10T00:14:27,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/95305558b81c4fe49f9b9141f703ef8c 2024-12-10T00:14:27,771 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e62b15f442814524b0a6bbe6e7b0b71a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=717, earliestPutTs=1733789664834 2024-12-10T00:14:27,771 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3afb8971ae9a4009946f22cde35f71ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789727766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789727770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789727771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789727774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789727780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/cbc151ed3f934094bdcf7bd5f2ea0214 is 50, key is test_row_0/B:col10/1733789667123/Put/seqid=0 2024-12-10T00:14:27,814 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:27,814 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/fb315b69aab74f108bb8628a4a703ad3 is 50, key is test_row_0/B:col10/1733789667013/Put/seqid=0 2024-12-10T00:14:27,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741921_1097 (size=12301) 2024-12-10T00:14:27,854 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/cbc151ed3f934094bdcf7bd5f2ea0214 2024-12-10T00:14:27,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789727880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2a349255e18648d385f78f5009260988 is 50, key is test_row_0/C:col10/1733789667123/Put/seqid=0 2024-12-10T00:14:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741922_1098 (size=13221) 2024-12-10T00:14:27,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789727890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:27,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:27,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741923_1099 (size=12301) 2024-12-10T00:14:28,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789728090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789728099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,154 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/499208a82c894e28bf2fa541c059ec78 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/499208a82c894e28bf2fa541c059ec78 2024-12-10T00:14:28,168 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 499208a82c894e28bf2fa541c059ec78(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:28,168 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:28,168 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789667622; duration=0sec 2024-12-10T00:14:28,168 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:28,168 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:28,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789728279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,291 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/fb315b69aab74f108bb8628a4a703ad3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fb315b69aab74f108bb8628a4a703ad3 2024-12-10T00:14:28,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789728290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789728292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,299 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into fb315b69aab74f108bb8628a4a703ad3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:28,299 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:28,299 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789667622; duration=0sec 2024-12-10T00:14:28,299 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:28,299 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:28,307 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=755 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2a349255e18648d385f78f5009260988 2024-12-10T00:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/95305558b81c4fe49f9b9141f703ef8c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c 2024-12-10T00:14:28,321 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c, entries=150, sequenceid=755, filesize=12.0 K 2024-12-10T00:14:28,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/cbc151ed3f934094bdcf7bd5f2ea0214 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214 2024-12-10T00:14:28,330 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214, entries=150, sequenceid=755, filesize=12.0 K 2024-12-10T00:14:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2a349255e18648d385f78f5009260988 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988 2024-12-10T00:14:28,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988, entries=150, sequenceid=755, filesize=12.0 K 2024-12-10T00:14:28,339 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~401.91 KB/411552, currentSize=40.25 KB/41220 for a5f283ed44d3b86dba39726447c58806 in 657ms, sequenceid=755, compaction requested=false 2024-12-10T00:14:28,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:28,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:28,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-10T00:14:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-10T00:14:28,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-10T00:14:28,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3100 sec 2024-12-10T00:14:28,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3210 sec 2024-12-10T00:14:28,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:28,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d74929c317ab4c4cb35f1ca9f3d43f0e is 50, key is test_row_0/A:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:28,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741924_1100 (size=14741) 2024-12-10T00:14:28,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d74929c317ab4c4cb35f1ca9f3d43f0e 2024-12-10T00:14:28,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/db1a4383630446f68504359c9e1d3233 is 50, key is test_row_0/B:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741925_1101 (size=12301) 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789668468, max timestamp=1733789668481Num uniques 859; 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789668468, max timestamp=1733789668481Num uniques 859; 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789668468, max timestamp=1733789668481Num uniques 859; 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:14:28,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:14:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789728521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789728523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789728629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789728640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789728840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789728850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:28,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/db1a4383630446f68504359c9e1d3233 2024-12-10T00:14:28,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2c16f8f23fc94de491b74e39a6fee279 is 50, key is test_row_0/C:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:28,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741926_1102 (size=12301) 2024-12-10T00:14:29,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789729160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789729160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T00:14:29,180 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:29,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-10T00:14:29,183 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:29,184 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T00:14:29,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T00:14:29,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=768 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2c16f8f23fc94de491b74e39a6fee279 2024-12-10T00:14:29,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789729300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,302 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/d74929c317ab4c4cb35f1ca9f3d43f0e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e 2024-12-10T00:14:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789729310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789729310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,316 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,317 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e, entries=200, sequenceid=768, filesize=14.4 K 2024-12-10T00:14:29,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/db1a4383630446f68504359c9e1d3233 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233 2024-12-10T00:14:29,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233, entries=150, sequenceid=768, filesize=12.0 K 2024-12-10T00:14:29,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/2c16f8f23fc94de491b74e39a6fee279 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279 2024-12-10T00:14:29,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T00:14:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:29,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279, entries=150, sequenceid=768, filesize=12.0 K 2024-12-10T00:14:29,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a5f283ed44d3b86dba39726447c58806 in 935ms, sequenceid=768, compaction requested=true 2024-12-10T00:14:29,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:29,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:29,341 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:29,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:29,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:29,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:29,342 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:29,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:29,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:29,343 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:29,343 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:29,343 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,343 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/499208a82c894e28bf2fa541c059ec78, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=39.3 K 2024-12-10T00:14:29,344 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:29,344 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:29,344 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,344 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 499208a82c894e28bf2fa541c059ec78, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:29,344 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fb315b69aab74f108bb8628a4a703ad3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.9 K 2024-12-10T00:14:29,344 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95305558b81c4fe49f9b9141f703ef8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733789667123 2024-12-10T00:14:29,345 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fb315b69aab74f108bb8628a4a703ad3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:29,345 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d74929c317ab4c4cb35f1ca9f3d43f0e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:29,345 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc151ed3f934094bdcf7bd5f2ea0214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733789667123 2024-12-10T00:14:29,346 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting db1a4383630446f68504359c9e1d3233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:29,354 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:29,355 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/f66178b144fe4ddeac56816eb28f55b7 is 50, key is test_row_0/A:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:29,363 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:29,363 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/8681f87d4ba74bd3bacf527b15e8d2ca is 50, key is test_row_0/B:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741927_1103 (size=13323) 2024-12-10T00:14:29,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741928_1104 (size=13323) 2024-12-10T00:14:29,385 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/8681f87d4ba74bd3bacf527b15e8d2ca as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8681f87d4ba74bd3bacf527b15e8d2ca 2024-12-10T00:14:29,395 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 8681f87d4ba74bd3bacf527b15e8d2ca(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:29,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:29,396 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789669341; duration=0sec 2024-12-10T00:14:29,396 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:29,396 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:29,396 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:29,403 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:29,403 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:29,403 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,403 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0999969158df4798b13eb40b7e85a600, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=36.9 K 2024-12-10T00:14:29,404 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0999969158df4798b13eb40b7e85a600, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=728, earliestPutTs=1733789667013 2024-12-10T00:14:29,404 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a349255e18648d385f78f5009260988, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=755, earliestPutTs=1733789667123 2024-12-10T00:14:29,405 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c16f8f23fc94de491b74e39a6fee279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:29,427 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:29,428 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/425279e0c4734deabbfa300aa26e8bd7 is 50, key is test_row_0/C:col10/1733789667755/Put/seqid=0 2024-12-10T00:14:29,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741929_1105 (size=13323) 2024-12-10T00:14:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T00:14:29,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T00:14:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:29,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T00:14:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ee4309057110420088fbb859baee23f3 is 50, key is test_row_0/A:col10/1733789668518/Put/seqid=0 2024-12-10T00:14:29,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741930_1106 (size=12301) 2024-12-10T00:14:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:29,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789729692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789729693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,775 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/f66178b144fe4ddeac56816eb28f55b7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f66178b144fe4ddeac56816eb28f55b7 2024-12-10T00:14:29,785 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into f66178b144fe4ddeac56816eb28f55b7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:29,785 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:29,785 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789669341; duration=0sec 2024-12-10T00:14:29,786 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:29,786 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:29,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789729799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789729799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T00:14:29,852 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/425279e0c4734deabbfa300aa26e8bd7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/425279e0c4734deabbfa300aa26e8bd7 2024-12-10T00:14:29,859 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 425279e0c4734deabbfa300aa26e8bd7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:29,859 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:29,859 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789669342; duration=0sec 2024-12-10T00:14:29,860 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:29,860 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:29,933 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=805 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ee4309057110420088fbb859baee23f3 2024-12-10T00:14:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/66af1ea165cc46ea979572f283f49b6d is 50, key is test_row_0/B:col10/1733789668518/Put/seqid=0 2024-12-10T00:14:29,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741931_1107 (size=12301) 2024-12-10T00:14:29,971 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=805 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/66af1ea165cc46ea979572f283f49b6d 2024-12-10T00:14:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/258fbddc0a054d718646735450f67983 is 50, key is test_row_0/C:col10/1733789668518/Put/seqid=0 2024-12-10T00:14:29,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741932_1108 (size=12301) 2024-12-10T00:14:29,991 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=805 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/258fbddc0a054d718646735450f67983 2024-12-10T00:14:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/ee4309057110420088fbb859baee23f3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3 2024-12-10T00:14:30,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3, entries=150, sequenceid=805, filesize=12.0 K 2024-12-10T00:14:30,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/66af1ea165cc46ea979572f283f49b6d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d 2024-12-10T00:14:30,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789730010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789730010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,014 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d, entries=150, sequenceid=805, filesize=12.0 K 2024-12-10T00:14:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/258fbddc0a054d718646735450f67983 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983 2024-12-10T00:14:30,024 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983, entries=150, sequenceid=805, filesize=12.0 K 2024-12-10T00:14:30,026 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for a5f283ed44d3b86dba39726447c58806 in 525ms, sequenceid=805, compaction requested=false 2024-12-10T00:14:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-10T00:14:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-10T00:14:30,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-10T00:14:30,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 845 msec 2024-12-10T00:14:30,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 854 msec 2024-12-10T00:14:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T00:14:30,320 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:30,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:30,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T00:14:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-10T00:14:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:30,330 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:30,332 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:30,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:30,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/f26ab0cbca244f978be35c89d2d4bc6e is 50, key is test_row_0/A:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:30,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741933_1109 (size=14741) 2024-12-10T00:14:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789730391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789730391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:30,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T00:14:30,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:30,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789730509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789730510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T00:14:30,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:30,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789730719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789730720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:30,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/f26ab0cbca244f978be35c89d2d4bc6e 2024-12-10T00:14:30,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/96a4018affa1486caa91c44c2d6b4ff3 is 50, key is test_row_0/B:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:30,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741934_1110 (size=12301) 2024-12-10T00:14:30,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/96a4018affa1486caa91c44c2d6b4ff3 2024-12-10T00:14:30,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/4ce6a6ffcaaf4441b249672040d3ffc3 is 50, key is test_row_0/C:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:30,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:30,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741935_1111 (size=12301) 2024-12-10T00:14:30,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=820 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/4ce6a6ffcaaf4441b249672040d3ffc3 2024-12-10T00:14:30,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/f26ab0cbca244f978be35c89d2d4bc6e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e 2024-12-10T00:14:30,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e, entries=200, sequenceid=820, filesize=14.4 K 2024-12-10T00:14:30,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/96a4018affa1486caa91c44c2d6b4ff3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3 2024-12-10T00:14:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3, entries=150, sequenceid=820, filesize=12.0 K 2024-12-10T00:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/4ce6a6ffcaaf4441b249672040d3ffc3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3 2024-12-10T00:14:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3, entries=150, sequenceid=820, filesize=12.0 K 2024-12-10T00:14:30,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a5f283ed44d3b86dba39726447c58806 in 532ms, sequenceid=820, compaction requested=true 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:30,854 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:30,854 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:30,855 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:30,855 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:30,855 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:30,855 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:30,855 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,855 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,855 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f66178b144fe4ddeac56816eb28f55b7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=39.4 K 2024-12-10T00:14:30,856 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8681f87d4ba74bd3bacf527b15e8d2ca, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=37.0 K 2024-12-10T00:14:30,856 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66178b144fe4ddeac56816eb28f55b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:30,856 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8681f87d4ba74bd3bacf527b15e8d2ca, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:30,856 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee4309057110420088fbb859baee23f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=805, earliestPutTs=1733789668502 2024-12-10T00:14:30,856 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 66af1ea165cc46ea979572f283f49b6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=805, earliestPutTs=1733789668502 2024-12-10T00:14:30,857 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f26ab0cbca244f978be35c89d2d4bc6e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:30,858 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a4018affa1486caa91c44c2d6b4ff3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:30,878 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:30,878 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:30,879 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/1ff3026824b444e68c96afb83f889abd is 50, key is test_row_0/A:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:30,879 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/80b77a4c89124ab18a6e15c4f71e7205 is 50, key is test_row_0/B:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:30,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741936_1112 (size=13425) 2024-12-10T00:14:30,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741937_1113 (size=13425) 2024-12-10T00:14:30,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:30,951 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:30,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:30,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4cd5323b0c344909a6171178961668b6 is 50, key is test_row_0/A:col10/1733789670388/Put/seqid=0 2024-12-10T00:14:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741938_1114 (size=12301) 2024-12-10T00:14:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:31,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. as already flushing 2024-12-10T00:14:31,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789731057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789731066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789731170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789731179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,307 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/80b77a4c89124ab18a6e15c4f71e7205 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/80b77a4c89124ab18a6e15c4f71e7205 2024-12-10T00:14:31,307 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/1ff3026824b444e68c96afb83f889abd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1ff3026824b444e68c96afb83f889abd 2024-12-10T00:14:31,318 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into 80b77a4c89124ab18a6e15c4f71e7205(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:31,318 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:31,318 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789670854; duration=0sec 2024-12-10T00:14:31,318 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:31,318 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:31,318 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:31,325 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:31,326 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:31,326 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:31,326 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/425279e0c4734deabbfa300aa26e8bd7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=37.0 K 2024-12-10T00:14:31,326 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 1ff3026824b444e68c96afb83f889abd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:31,326 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:31,326 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789670854; duration=0sec 2024-12-10T00:14:31,327 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:31,327 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:31,327 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 425279e0c4734deabbfa300aa26e8bd7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=768, earliestPutTs=1733789667755 2024-12-10T00:14:31,328 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 258fbddc0a054d718646735450f67983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=805, earliestPutTs=1733789668502 2024-12-10T00:14:31,328 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce6a6ffcaaf4441b249672040d3ffc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53582 deadline: 1733789731330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53542 deadline: 1733789731330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,336 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,337 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53550 deadline: 1733789731331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,339 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in a5f283ed44d3b86dba39726447c58806 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:31,340 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,341 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/e74dd31917a9400f9e527ab4073d0ad9 is 50, key is test_row_0/C:col10/1733789670321/Put/seqid=0 2024-12-10T00:14:31,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741939_1115 (size=13425) 2024-12-10T00:14:31,375 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/e74dd31917a9400f9e527ab4073d0ad9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/e74dd31917a9400f9e527ab4073d0ad9 2024-12-10T00:14:31,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789731380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,384 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into e74dd31917a9400f9e527ab4073d0ad9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:31,384 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:31,385 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789670854; duration=0sec 2024-12-10T00:14:31,385 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:31,385 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:31,387 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=839 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4cd5323b0c344909a6171178961668b6 2024-12-10T00:14:31,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789731390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/64e6c3c044904c96977f086d1c3bd880 is 50, key is test_row_0/B:col10/1733789670388/Put/seqid=0 2024-12-10T00:14:31,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741940_1116 (size=12301) 2024-12-10T00:14:31,415 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=839 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/64e6c3c044904c96977f086d1c3bd880 2024-12-10T00:14:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3337e1e5ac724145a40c47590c7c92a5 is 50, key is test_row_0/C:col10/1733789670388/Put/seqid=0 2024-12-10T00:14:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741941_1117 (size=12301) 2024-12-10T00:14:31,435 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=839 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3337e1e5ac724145a40c47590c7c92a5 2024-12-10T00:14:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/4cd5323b0c344909a6171178961668b6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6 2024-12-10T00:14:31,458 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6, entries=150, sequenceid=839, filesize=12.0 K 2024-12-10T00:14:31,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/64e6c3c044904c96977f086d1c3bd880 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880 2024-12-10T00:14:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:31,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880, entries=150, sequenceid=839, filesize=12.0 K 2024-12-10T00:14:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/3337e1e5ac724145a40c47590c7c92a5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5 2024-12-10T00:14:31,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5, entries=150, sequenceid=839, filesize=12.0 K 2024-12-10T00:14:31,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a5f283ed44d3b86dba39726447c58806 in 549ms, sequenceid=839, compaction requested=false 2024-12-10T00:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-10T00:14:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-10T00:14:31,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-10T00:14:31,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-12-10T00:14:31,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1860 sec 2024-12-10T00:14:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:31,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:14:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:31,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:31,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/6ec9f592736842f991649e1a302942b3 is 50, key is test_row_0/A:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:31,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741942_1118 (size=14741) 2024-12-10T00:14:31,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789731731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789731734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789731840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789731840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789732050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789732050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/6ec9f592736842f991649e1a302942b3 2024-12-10T00:14:32,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4bfe07ca15c246b494839d110019dec8 is 50, key is test_row_0/B:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:32,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741943_1119 (size=12301) 2024-12-10T00:14:32,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789732359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789732361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,438 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-10T00:14:32,438 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:32,439 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-10T00:14:32,439 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:32,443 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,443 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,444 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:32,444 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,445 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-10T00:14:32,445 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:32,445 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,445 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:32,446 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-10T00:14:32,446 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:32,446 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,446 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:32,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:32,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T00:14:32,482 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:32,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4bfe07ca15c246b494839d110019dec8 2024-12-10T00:14:32,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/559b89c6708449f9b053b831f07b474a is 50, key is test_row_0/C:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:32,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741944_1120 (size=12301) 2024-12-10T00:14:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53624 deadline: 1733789732869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53578 deadline: 1733789732869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:14:32,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/559b89c6708449f9b053b831f07b474a 2024-12-10T00:14:32,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/6ec9f592736842f991649e1a302942b3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3 2024-12-10T00:14:32,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3, entries=200, sequenceid=856, filesize=14.4 K 2024-12-10T00:14:32,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/4bfe07ca15c246b494839d110019dec8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8 2024-12-10T00:14:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8, entries=150, sequenceid=856, filesize=12.0 K 2024-12-10T00:14:32,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/559b89c6708449f9b053b831f07b474a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a 2024-12-10T00:14:32,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a, entries=150, sequenceid=856, filesize=12.0 K 2024-12-10T00:14:32,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a5f283ed44d3b86dba39726447c58806 in 1260ms, sequenceid=856, compaction requested=true 2024-12-10T00:14:32,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:32,953 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:32,953 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5f283ed44d3b86dba39726447c58806:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:32,954 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:32,954 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:32,954 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/A is initiating minor compaction (all files) 2024-12-10T00:14:32,954 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/B is initiating minor compaction (all files) 2024-12-10T00:14:32,954 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/B in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:32,954 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/A in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:32,954 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1ff3026824b444e68c96afb83f889abd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=39.5 K 2024-12-10T00:14:32,954 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/80b77a4c89124ab18a6e15c4f71e7205, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=37.1 K 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b77a4c89124ab18a6e15c4f71e7205, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff3026824b444e68c96afb83f889abd, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e6c3c044904c96977f086d1c3bd880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=839, earliestPutTs=1733789670379 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cd5323b0c344909a6171178961668b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=839, earliestPutTs=1733789670379 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec9f592736842f991649e1a302942b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733789671036 2024-12-10T00:14:32,955 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfe07ca15c246b494839d110019dec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733789671036 2024-12-10T00:14:32,964 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:32,964 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/d327e1aa9d354705b9c86fab81c9b96b is 50, key is test_row_0/B:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:32,964 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#A#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:32,965 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/995c5487b991475f93044f2f5874dd8c is 50, key is test_row_0/A:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:32,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741945_1121 (size=13527) 2024-12-10T00:14:32,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741946_1122 (size=13527) 2024-12-10T00:14:33,410 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/d327e1aa9d354705b9c86fab81c9b96b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d327e1aa9d354705b9c86fab81c9b96b 2024-12-10T00:14:33,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/995c5487b991475f93044f2f5874dd8c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/995c5487b991475f93044f2f5874dd8c 2024-12-10T00:14:33,416 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/B of a5f283ed44d3b86dba39726447c58806 into d327e1aa9d354705b9c86fab81c9b96b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:33,416 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/A of a5f283ed44d3b86dba39726447c58806 into 995c5487b991475f93044f2f5874dd8c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:33,416 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/B, priority=13, startTime=1733789672953; duration=0sec 2024-12-10T00:14:33,416 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/A, priority=13, startTime=1733789672953; duration=0sec 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:B 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:33,416 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:A 2024-12-10T00:14:33,417 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:33,417 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): a5f283ed44d3b86dba39726447c58806/C is initiating minor compaction (all files) 2024-12-10T00:14:33,417 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5f283ed44d3b86dba39726447c58806/C in TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:33,418 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/e74dd31917a9400f9e527ab4073d0ad9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp, totalSize=37.1 K 2024-12-10T00:14:33,418 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e74dd31917a9400f9e527ab4073d0ad9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=820, earliestPutTs=1733789669691 2024-12-10T00:14:33,418 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3337e1e5ac724145a40c47590c7c92a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=839, earliestPutTs=1733789670379 2024-12-10T00:14:33,419 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 559b89c6708449f9b053b831f07b474a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1733789671036 2024-12-10T00:14:33,426 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5f283ed44d3b86dba39726447c58806#C#compaction#110 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:33,427 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/56d645920cfb4e40a0e3756427c2fabe is 50, key is test_row_0/C:col10/1733789671690/Put/seqid=0 2024-12-10T00:14:33,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741947_1123 (size=13527) 2024-12-10T00:14:33,794 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:14:33,838 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/56d645920cfb4e40a0e3756427c2fabe as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/56d645920cfb4e40a0e3756427c2fabe 2024-12-10T00:14:33,854 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5f283ed44d3b86dba39726447c58806/C of a5f283ed44d3b86dba39726447c58806 into 56d645920cfb4e40a0e3756427c2fabe(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:33,854 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:33,854 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., storeName=a5f283ed44d3b86dba39726447c58806/C, priority=13, startTime=1733789672953; duration=0sec 2024-12-10T00:14:33,854 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:33,854 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5f283ed44d3b86dba39726447c58806:C 2024-12-10T00:14:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:33,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:33,884 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-10T00:14:33,884 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:33,884 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:33,884 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:33,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:33,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:33,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:33,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:33,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:33,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:33,887 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-10T00:14:33,887 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:33,887 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:33,887 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:33,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:33,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:33,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/8a27eaa92e024f0aa4af94e555bb6bc9 is 50, key is test_row_0/A:col10/1733789673880/Put/seqid=0 2024-12-10T00:14:33,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741948_1124 (size=12301) 2024-12-10T00:14:34,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=879 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/8a27eaa92e024f0aa4af94e555bb6bc9 2024-12-10T00:14:34,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b19ee98dfb044434aea107bb23fb8602 is 50, key is test_row_0/B:col10/1733789673880/Put/seqid=0 2024-12-10T00:14:34,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741949_1125 (size=12301) 2024-12-10T00:14:34,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=879 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b19ee98dfb044434aea107bb23fb8602 2024-12-10T00:14:34,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/12ab82b0dd744317ab26630a69e74570 is 50, key is test_row_0/C:col10/1733789673880/Put/seqid=0 2024-12-10T00:14:34,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741950_1126 (size=12301) 2024-12-10T00:14:34,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=879 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/12ab82b0dd744317ab26630a69e74570 2024-12-10T00:14:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/8a27eaa92e024f0aa4af94e555bb6bc9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/8a27eaa92e024f0aa4af94e555bb6bc9 2024-12-10T00:14:34,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/8a27eaa92e024f0aa4af94e555bb6bc9, entries=150, sequenceid=879, filesize=12.0 K 2024-12-10T00:14:34,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b19ee98dfb044434aea107bb23fb8602 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b19ee98dfb044434aea107bb23fb8602 2024-12-10T00:14:34,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b19ee98dfb044434aea107bb23fb8602, entries=150, sequenceid=879, filesize=12.0 K 2024-12-10T00:14:34,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/12ab82b0dd744317ab26630a69e74570 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/12ab82b0dd744317ab26630a69e74570 2024-12-10T00:14:34,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/12ab82b0dd744317ab26630a69e74570, entries=150, sequenceid=879, filesize=12.0 K 2024-12-10T00:14:34,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for a5f283ed44d3b86dba39726447c58806 in 497ms, sequenceid=879, compaction requested=false 2024-12-10T00:14:34,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5f283ed44d3b86dba39726447c58806: 2024-12-10T00:14:35,361 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-10T00:14:35,361 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:35,361 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,361 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:35,371 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-10T00:14:35,371 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-10T00:14:35,372 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:35,372 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:14:35,372 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,372 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,372 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,372 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:14:35,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 160 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 155 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2370 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2399 2024-12-10T00:14:35,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:14:35,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-12-10T00:14:35,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3471 rows 2024-12-10T00:14:35,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-12-10T00:14:35,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3442 rows 2024-12-10T00:14:35,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:14:35,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:14:35,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:35,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:14:35,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:14:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:35,384 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789675384"}]},"ts":"1733789675384"} 2024-12-10T00:14:35,386 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:14:35,386 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:14:35,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:14:35,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, UNASSIGN}] 2024-12-10T00:14:35,394 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, UNASSIGN 2024-12-10T00:14:35,395 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a5f283ed44d3b86dba39726447c58806, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:35,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, UNASSIGN because future has completed 2024-12-10T00:14:35,398 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:14:35,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:35,555 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:35,555 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing a5f283ed44d3b86dba39726447c58806, disabling compactions & flushes 2024-12-10T00:14:35,556 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. after waiting 0 ms 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:35,556 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing a5f283ed44d3b86dba39726447c58806 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=A 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=B 2024-12-10T00:14:35,556 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:35,557 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5f283ed44d3b86dba39726447c58806, store=C 2024-12-10T00:14:35,557 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:35,560 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2857804cbc3843549ecc7f9f879b4fad is 50, key is test_row_0/A:col10/1733789675370/Put/seqid=0 2024-12-10T00:14:35,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741951_1127 (size=9857) 2024-12-10T00:14:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:35,965 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=885 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2857804cbc3843549ecc7f9f879b4fad 2024-12-10T00:14:35,973 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b65d5e3b4ff74190b869a89649f654a3 is 50, key is test_row_0/B:col10/1733789675370/Put/seqid=0 2024-12-10T00:14:35,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741952_1128 (size=9857) 2024-12-10T00:14:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:36,379 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=885 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b65d5e3b4ff74190b869a89649f654a3 2024-12-10T00:14:36,386 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/0c6205376fc54b668b72285a71161341 is 50, key is test_row_0/C:col10/1733789675370/Put/seqid=0 2024-12-10T00:14:36,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741953_1129 (size=9857) 2024-12-10T00:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:36,790 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=885 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/0c6205376fc54b668b72285a71161341 2024-12-10T00:14:36,796 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/A/2857804cbc3843549ecc7f9f879b4fad as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2857804cbc3843549ecc7f9f879b4fad 2024-12-10T00:14:36,801 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2857804cbc3843549ecc7f9f879b4fad, entries=100, sequenceid=885, filesize=9.6 K 2024-12-10T00:14:36,802 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/B/b65d5e3b4ff74190b869a89649f654a3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b65d5e3b4ff74190b869a89649f654a3 2024-12-10T00:14:36,807 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b65d5e3b4ff74190b869a89649f654a3, entries=100, sequenceid=885, filesize=9.6 K 2024-12-10T00:14:36,809 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/.tmp/C/0c6205376fc54b668b72285a71161341 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0c6205376fc54b668b72285a71161341 2024-12-10T00:14:36,814 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0c6205376fc54b668b72285a71161341, entries=100, sequenceid=885, filesize=9.6 K 2024-12-10T00:14:36,815 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a5f283ed44d3b86dba39726447c58806 in 1259ms, sequenceid=885, compaction requested=true 2024-12-10T00:14:36,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/0d93aa02831b4a31a27e757ef07efb15, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fb5930abf9f5407e91526c39230ab241, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/aa9b2c4521a740d7bf5a2168c0d54ad7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/399448e048dd473380bf9517fe18a3b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2414ca2b554d4f608d7da3f83800b30d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fbfbb4cc3e98438593c17d182ee8548c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d77cc21ec1284c11ad45361db24ac8ba, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/98889acfec404eedaa9e6241af7daebf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/499208a82c894e28bf2fa541c059ec78, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f66178b144fe4ddeac56816eb28f55b7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1ff3026824b444e68c96afb83f889abd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3] to archive 2024-12-10T00:14:36,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:14:36,826 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/0d93aa02831b4a31a27e757ef07efb15 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/0d93aa02831b4a31a27e757ef07efb15 2024-12-10T00:14:36,826 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fb5930abf9f5407e91526c39230ab241 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fb5930abf9f5407e91526c39230ab241 2024-12-10T00:14:36,827 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/7adf6fadf6d2403194e4bf19c166d0c6 2024-12-10T00:14:36,826 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fa154560116b48b0b27e9c60a33378cb 2024-12-10T00:14:36,827 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/bebe78d2f80f4d03bac9e6de2a865ffa 2024-12-10T00:14:36,827 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fec0adc0da8a448c90bbc69b262edb15 2024-12-10T00:14:36,827 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e569ce3b4641472589924ea003571b8e 2024-12-10T00:14:36,828 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fd1ed9c91bff476f8854c06da3cc7f3b 2024-12-10T00:14:36,829 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/399448e048dd473380bf9517fe18a3b4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/399448e048dd473380bf9517fe18a3b4 2024-12-10T00:14:36,829 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/05ae9a1ae3324af38513be2d85f9c183 2024-12-10T00:14:36,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ad8f12b808bd454bafdc9e30b7b6280b 2024-12-10T00:14:36,830 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5b7b2ee70c0f4994bf663c142e1450d2 2024-12-10T00:14:36,830 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/aa9b2c4521a740d7bf5a2168c0d54ad7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/aa9b2c4521a740d7bf5a2168c0d54ad7 2024-12-10T00:14:36,830 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/e53ef60dbb6146d18fc815dd86224585 2024-12-10T00:14:36,831 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/c80ac8542d2e4b04a797fb4f1f0ff9ce 2024-12-10T00:14:36,833 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2414ca2b554d4f608d7da3f83800b30d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2414ca2b554d4f608d7da3f83800b30d 2024-12-10T00:14:36,834 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/5361022ccca84e3094792bc69d3acf20 2024-12-10T00:14:36,834 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1e11adaaddb74521878abd0f1f780bb1 2024-12-10T00:14:36,834 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fbfbb4cc3e98438593c17d182ee8548c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/fbfbb4cc3e98438593c17d182ee8548c 2024-12-10T00:14:36,834 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2bd929a529c9495c8134cd36a66a9817 2024-12-10T00:14:36,835 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d77cc21ec1284c11ad45361db24ac8ba to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d77cc21ec1284c11ad45361db24ac8ba 2024-12-10T00:14:36,835 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a577d51186fe4f6ab2acc245bb9c0f28 2024-12-10T00:14:36,835 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d7542e5ba7fa4124be21c418d748e5f7 2024-12-10T00:14:36,836 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4ab279c0eeb6457a8df4ae0502019537 2024-12-10T00:14:36,836 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/a6248bab15a542978e842863bd2230fc 2024-12-10T00:14:36,836 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/98889acfec404eedaa9e6241af7daebf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/98889acfec404eedaa9e6241af7daebf 2024-12-10T00:14:36,837 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/499208a82c894e28bf2fa541c059ec78 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/499208a82c894e28bf2fa541c059ec78 2024-12-10T00:14:36,837 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/9533b8fb7f984dcc96f078394fc2c20e 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1ff3026824b444e68c96afb83f889abd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/1ff3026824b444e68c96afb83f889abd 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/d74929c317ab4c4cb35f1ca9f3d43f0e 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/ee4309057110420088fbb859baee23f3 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f26ab0cbca244f978be35c89d2d4bc6e 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f66178b144fe4ddeac56816eb28f55b7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/f66178b144fe4ddeac56816eb28f55b7 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/95305558b81c4fe49f9b9141f703ef8c 2024-12-10T00:14:36,839 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/4cd5323b0c344909a6171178961668b6 2024-12-10T00:14:36,840 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/6ec9f592736842f991649e1a302942b3 2024-12-10T00:14:36,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/eb6710dcbe0d460b87dc3f097a22d6ad, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/88431fcdb4cf4878adda012fa12b42e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4d4b9e70e362407291e23319c8cdca72, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/f507560040d747c68259f1dffcac9353, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/23a620edf7ac47d89762c0c98db6bbc3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4ea69405161f4b24b886634cd6d0d737, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d18363fe9d38407292a12a9081dfe7c2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/880da7b92d624b42b4fc4cd391b6c5a2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fb315b69aab74f108bb8628a4a703ad3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8681f87d4ba74bd3bacf527b15e8d2ca, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/80b77a4c89124ab18a6e15c4f71e7205, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8] to archive 2024-12-10T00:14:36,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:14:36,859 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/0cade55df5214080af4aba8dc2dedc1c 2024-12-10T00:14:36,859 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1eedd6ce81de455a8646244969b7d36c 2024-12-10T00:14:36,859 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/eb6710dcbe0d460b87dc3f097a22d6ad to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/eb6710dcbe0d460b87dc3f097a22d6ad 2024-12-10T00:14:36,860 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/93de7997f98e43c39402b648b914bcab 2024-12-10T00:14:36,860 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4de3bb6047ee4007b57e666f07960e25 2024-12-10T00:14:36,860 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/453f7b3bcbc343a1af621c8437a10e13 2024-12-10T00:14:36,860 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/611b26482d494e9e95099e014a5b044a 2024-12-10T00:14:36,861 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/88431fcdb4cf4878adda012fa12b42e1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/88431fcdb4cf4878adda012fa12b42e1 2024-12-10T00:14:36,864 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4d4b9e70e362407291e23319c8cdca72 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4d4b9e70e362407291e23319c8cdca72 2024-12-10T00:14:36,864 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fea5c11de74748d6a33709b37280aa45 2024-12-10T00:14:36,865 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/528d96d7cbca44f186509fd50329fb49 2024-12-10T00:14:36,865 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/5c0a67ec7c064632a25084f76ad73e8e 2024-12-10T00:14:36,866 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/f507560040d747c68259f1dffcac9353 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/f507560040d747c68259f1dffcac9353 2024-12-10T00:14:36,866 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/23a620edf7ac47d89762c0c98db6bbc3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/23a620edf7ac47d89762c0c98db6bbc3 2024-12-10T00:14:36,866 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/7599aff1437f4bb88124cdcb910a0cfe 2024-12-10T00:14:36,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/bc5f67cb4ab345498075bcef43f81fc7 2024-12-10T00:14:36,867 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4ea69405161f4b24b886634cd6d0d737 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4ea69405161f4b24b886634cd6d0d737 2024-12-10T00:14:36,868 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/1530ced4cfdd46a1951719739bb7d7e5 2024-12-10T00:14:36,868 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/54f85ef23b0c4842b390c014a003eb6d 2024-12-10T00:14:36,868 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d18363fe9d38407292a12a9081dfe7c2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d18363fe9d38407292a12a9081dfe7c2 2024-12-10T00:14:36,869 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/880da7b92d624b42b4fc4cd391b6c5a2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/880da7b92d624b42b4fc4cd391b6c5a2 2024-12-10T00:14:36,869 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8f8ff537abb8427cb4ffa76453507f4c 2024-12-10T00:14:36,870 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/36b275fe595d4f26b8b6a3aff46901bd 2024-12-10T00:14:36,870 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/2e5aafb50b8d49d9aebcd0a95ec06ae1 2024-12-10T00:14:36,874 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e62b15f442814524b0a6bbe6e7b0b71a 2024-12-10T00:14:36,874 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/e8adaf00cc794900b93785974ebb14d8 2024-12-10T00:14:36,875 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/3afb8971ae9a4009946f22cde35f71ff 2024-12-10T00:14:36,875 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fb315b69aab74f108bb8628a4a703ad3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/fb315b69aab74f108bb8628a4a703ad3 2024-12-10T00:14:36,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/cbc151ed3f934094bdcf7bd5f2ea0214 2024-12-10T00:14:36,876 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8681f87d4ba74bd3bacf527b15e8d2ca to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/8681f87d4ba74bd3bacf527b15e8d2ca 2024-12-10T00:14:36,876 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/db1a4383630446f68504359c9e1d3233 2024-12-10T00:14:36,876 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/66af1ea165cc46ea979572f283f49b6d 2024-12-10T00:14:36,877 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/80b77a4c89124ab18a6e15c4f71e7205 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/80b77a4c89124ab18a6e15c4f71e7205 2024-12-10T00:14:36,878 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/96a4018affa1486caa91c44c2d6b4ff3 2024-12-10T00:14:36,878 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/64e6c3c044904c96977f086d1c3bd880 2024-12-10T00:14:36,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/4bfe07ca15c246b494839d110019dec8 2024-12-10T00:14:36,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6f5873262c7a4147930cd872f54762ec, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/100d5c83bdbc469696fdcd69f6317d45, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/575064fda45c463e9ad4d4cb086c052a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/9436527a4cb5486e9680b21689f2b6cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b67896019cd24dd59eb3e44634fc7fb4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b6290186fa7547348e6be84dd0b06e90, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29c0dbcaeaaa4cce880660fe01d1b643, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/aae437500a5e43fda716c8c0e288b25a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0999969158df4798b13eb40b7e85a600, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/425279e0c4734deabbfa300aa26e8bd7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/e74dd31917a9400f9e527ab4073d0ad9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a] to archive 2024-12-10T00:14:36,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/100d5c83bdbc469696fdcd69f6317d45 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/100d5c83bdbc469696fdcd69f6317d45 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6f5873262c7a4147930cd872f54762ec to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6f5873262c7a4147930cd872f54762ec 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/c3659f8bc7664b5985f331586f5a4037 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ec092a05138643ada3e67d64af869999 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/fc871e5e865f431da8b80ba8b6fc6f8c 2024-12-10T00:14:36,891 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/64319796ef5c412cbab574254a1fd6c4 2024-12-10T00:14:36,892 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/90f8dd155da64741be79ee1fa4db8d59 2024-12-10T00:14:36,892 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ae02b6eb887649d8bf7db6000c94111e 2024-12-10T00:14:36,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/f8e8fda4bdbf42f6924989f4927f5c2e 2024-12-10T00:14:36,894 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/6b719866c3194dbdab93d2e3f0f04289 2024-12-10T00:14:36,894 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a901694f67eb4a94a6f1d310e1a8eab9 2024-12-10T00:14:36,894 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/9436527a4cb5486e9680b21689f2b6cf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/9436527a4cb5486e9680b21689f2b6cf 2024-12-10T00:14:36,894 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/8900b1d013f34d7291fda20a42083416 2024-12-10T00:14:36,895 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/84413c26a52c4549a6ca70aa9d818933 2024-12-10T00:14:36,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b67896019cd24dd59eb3e44634fc7fb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b67896019cd24dd59eb3e44634fc7fb4 2024-12-10T00:14:36,895 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/575064fda45c463e9ad4d4cb086c052a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/575064fda45c463e9ad4d4cb086c052a 2024-12-10T00:14:36,896 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/181c9c3e19a44510bc11cdf0105a96eb 2024-12-10T00:14:36,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3eb5022af6874fafbbad5a9c33aa1c2e 2024-12-10T00:14:36,896 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/a60b614bb4d94a77b196896d20e7ebcd 2024-12-10T00:14:36,897 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b6290186fa7547348e6be84dd0b06e90 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/b6290186fa7547348e6be84dd0b06e90 2024-12-10T00:14:36,897 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29c0dbcaeaaa4cce880660fe01d1b643 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29c0dbcaeaaa4cce880660fe01d1b643 2024-12-10T00:14:36,897 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/aae437500a5e43fda716c8c0e288b25a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/aae437500a5e43fda716c8c0e288b25a 2024-12-10T00:14:36,897 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/5ec208f5a4fa4db0bde66d70e193b394 2024-12-10T00:14:36,899 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ac422a221cc44b96a14dce5f51d536c4 2024-12-10T00:14:36,899 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/918b5bd4c0e44f268ff03936f5661de3 2024-12-10T00:14:36,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/29e1a8292837476ba35bcae1a9e43d89 2024-12-10T00:14:36,899 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/ef1b9f9c80084a13ad38a49bce049932 2024-12-10T00:14:36,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2a349255e18648d385f78f5009260988 2024-12-10T00:14:36,900 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0999969158df4798b13eb40b7e85a600 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0999969158df4798b13eb40b7e85a600 2024-12-10T00:14:36,900 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/425279e0c4734deabbfa300aa26e8bd7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/425279e0c4734deabbfa300aa26e8bd7 2024-12-10T00:14:36,901 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/2c16f8f23fc94de491b74e39a6fee279 2024-12-10T00:14:36,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/258fbddc0a054d718646735450f67983 2024-12-10T00:14:36,901 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/4ce6a6ffcaaf4441b249672040d3ffc3 2024-12-10T00:14:36,902 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/559b89c6708449f9b053b831f07b474a 2024-12-10T00:14:36,902 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/e74dd31917a9400f9e527ab4073d0ad9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/e74dd31917a9400f9e527ab4073d0ad9 2024-12-10T00:14:36,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/3337e1e5ac724145a40c47590c7c92a5 2024-12-10T00:14:36,907 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/recovered.edits/888.seqid, newMaxSeqId=888, maxSeqId=1 2024-12-10T00:14:36,908 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. 2024-12-10T00:14:36,908 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for a5f283ed44d3b86dba39726447c58806: Waiting for close lock at 1733789675556Running coprocessor pre-close hooks at 1733789675556Disabling compacts and flushes for region at 1733789675556Disabling writes for close at 1733789675556Obtaining lock to block concurrent updates at 1733789675556Preparing flush snapshotting stores in a5f283ed44d3b86dba39726447c58806 at 1733789675556Finished memstore snapshotting TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733789675557 (+1 ms)Flushing stores of TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806. at 1733789675557Flushing a5f283ed44d3b86dba39726447c58806/A: creating writer at 1733789675557Flushing a5f283ed44d3b86dba39726447c58806/A: appending metadata at 1733789675560 (+3 ms)Flushing a5f283ed44d3b86dba39726447c58806/A: closing flushed file at 1733789675560Flushing a5f283ed44d3b86dba39726447c58806/B: creating writer at 1733789675970 (+410 ms)Flushing a5f283ed44d3b86dba39726447c58806/B: appending metadata at 1733789675973 (+3 ms)Flushing a5f283ed44d3b86dba39726447c58806/B: closing flushed file at 1733789675973Flushing a5f283ed44d3b86dba39726447c58806/C: creating writer at 1733789676383 (+410 ms)Flushing a5f283ed44d3b86dba39726447c58806/C: appending metadata at 1733789676386 (+3 ms)Flushing a5f283ed44d3b86dba39726447c58806/C: closing flushed file at 1733789676386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50e5a12d: reopening flushed file at 1733789676795 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a4663a1: reopening flushed file at 1733789676801 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@314a5e06: reopening flushed file at 1733789676808 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a5f283ed44d3b86dba39726447c58806 in 1259ms, sequenceid=885, compaction requested=true at 1733789676815 (+7 ms)Writing region close event to WAL at 1733789676903 (+88 ms)Running coprocessor post-close hooks at 1733789676907 (+4 ms)Closed at 1733789676908 (+1 ms) 2024-12-10T00:14:36,910 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:36,911 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a5f283ed44d3b86dba39726447c58806, regionState=CLOSED 2024-12-10T00:14:36,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:14:36,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-10T00:14:36,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure a5f283ed44d3b86dba39726447c58806, server=a67c4886b4f7,36911,1733789645811 in 1.5160 sec 2024-12-10T00:14:36,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-10T00:14:36,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5f283ed44d3b86dba39726447c58806, UNASSIGN in 1.5220 sec 2024-12-10T00:14:36,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-10T00:14:36,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5310 sec 2024-12-10T00:14:36,922 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789676922"}]},"ts":"1733789676922"} 2024-12-10T00:14:36,924 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:14:36,924 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:14:36,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5470 sec 2024-12-10T00:14:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-10T00:14:37,530 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:37,533 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:14:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,538 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-10T00:14:37,539 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,543 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:37,547 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/recovered.edits] 2024-12-10T00:14:37,551 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/995c5487b991475f93044f2f5874dd8c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/995c5487b991475f93044f2f5874dd8c 2024-12-10T00:14:37,551 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2857804cbc3843549ecc7f9f879b4fad to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/2857804cbc3843549ecc7f9f879b4fad 2024-12-10T00:14:37,551 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/8a27eaa92e024f0aa4af94e555bb6bc9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/A/8a27eaa92e024f0aa4af94e555bb6bc9 2024-12-10T00:14:37,554 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d327e1aa9d354705b9c86fab81c9b96b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/d327e1aa9d354705b9c86fab81c9b96b 2024-12-10T00:14:37,554 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b19ee98dfb044434aea107bb23fb8602 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b19ee98dfb044434aea107bb23fb8602 2024-12-10T00:14:37,554 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b65d5e3b4ff74190b869a89649f654a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/B/b65d5e3b4ff74190b869a89649f654a3 2024-12-10T00:14:37,558 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/12ab82b0dd744317ab26630a69e74570 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/12ab82b0dd744317ab26630a69e74570 2024-12-10T00:14:37,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0c6205376fc54b668b72285a71161341 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/0c6205376fc54b668b72285a71161341 2024-12-10T00:14:37,558 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/56d645920cfb4e40a0e3756427c2fabe to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/C/56d645920cfb4e40a0e3756427c2fabe 2024-12-10T00:14:37,562 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/recovered.edits/888.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806/recovered.edits/888.seqid 2024-12-10T00:14:37,563 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/a5f283ed44d3b86dba39726447c58806 2024-12-10T00:14:37,563 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:14:37,568 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T00:14:37,580 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:14:37,615 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:14:37,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:14:37,619 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789677619"}]},"ts":"9223372036854775807"} 2024-12-10T00:14:37,627 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:14:37,627 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a5f283ed44d3b86dba39726447c58806, NAME => 'TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:14:37,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:14:37,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789677628"}]},"ts":"9223372036854775807"} 2024-12-10T00:14:37,634 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:14:37,635 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 101 msec 2024-12-10T00:14:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-10T00:14:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:14:37,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:37,665 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=256 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1766026422_22 at /127.0.0.1:40594 [Waiting for operation #258] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1976302026_22 at /127.0.0.1:38234 [Waiting for operation #240] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a67c4886b4f7:36911-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1976302026_22 at /127.0.0.1:38232 [Waiting for operation #242] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 314) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3403 (was 4840) 2024-12-10T00:14:37,675 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=256, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=3403 2024-12-10T00:14:37,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:14:37,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:14:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:37,681 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:14:37,681 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:37,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-10T00:14:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:37,683 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:14:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741954_1130 (size=1061) 2024-12-10T00:14:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:38,094 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:14:38,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741955_1131 (size=53) 2024-12-10T00:14:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ce74af60a20cd73f95a0f2738491f1a3, disabling compactions & flushes 2024-12-10T00:14:38,502 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. after waiting 0 ms 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,502 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,502 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce74af60a20cd73f95a0f2738491f1a3: Waiting for close lock at 1733789678502Disabling compacts and flushes for region at 1733789678502Disabling writes for close at 1733789678502Writing region close event to WAL at 1733789678502Closed at 1733789678502 2024-12-10T00:14:38,503 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:14:38,503 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789678503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789678503"}]},"ts":"1733789678503"} 2024-12-10T00:14:38,505 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:14:38,506 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:14:38,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789678506"}]},"ts":"1733789678506"} 2024-12-10T00:14:38,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:14:38,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, ASSIGN}] 2024-12-10T00:14:38,509 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, ASSIGN 2024-12-10T00:14:38,509 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:14:38,660 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:38,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, ASSIGN because future has completed 2024-12-10T00:14:38,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:38,818 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:38,820 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,821 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:38,822 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName A 2024-12-10T00:14:38,822 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:38,823 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:38,823 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,824 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:38,824 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName B 2024-12-10T00:14:38,824 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:38,825 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:38,825 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,826 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:38,826 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName C 2024-12-10T00:14:38,826 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:38,827 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:38,827 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,827 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,828 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,828 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,830 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,830 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,830 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:14:38,831 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,834 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:14:38,834 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened ce74af60a20cd73f95a0f2738491f1a3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67094582, jitterRate=-2.1281838417053223E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:14:38,834 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:38,835 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for ce74af60a20cd73f95a0f2738491f1a3: Running coprocessor pre-open hook at 1733789678818Writing region info on filesystem at 1733789678818Initializing all the Stores at 1733789678819 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789678819Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789678820 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789678820Cleaning up temporary data from old regions at 1733789678830 (+10 ms)Running coprocessor post-open hooks at 1733789678834 (+4 ms)Region opened successfully at 1733789678835 (+1 ms) 2024-12-10T00:14:38,836 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., pid=34, masterSystemTime=1733789678814 2024-12-10T00:14:38,839 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,839 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:38,839 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:38,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:14:38,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-10T00:14:38,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 in 180 msec 2024-12-10T00:14:38,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-10T00:14:38,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, ASSIGN in 336 msec 2024-12-10T00:14:38,847 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:14:38,847 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789678847"}]},"ts":"1733789678847"} 2024-12-10T00:14:38,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:14:38,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:14:38,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-10T00:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-10T00:14:39,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:39,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab3237a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:39,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:39,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:39,832 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28df810d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:39,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:39,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa3bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:39,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:39,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:39,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:39,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:39,841 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:14:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:14:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b535217 2024-12-10T00:14:39,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:14:39,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:14:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:14:39,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:14:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T00:14:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741956_1132 (size=1097) 2024-12-10T00:14:39,870 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T00:14:39,870 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T00:14:39,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:14:39,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE}] 2024-12-10T00:14:39,884 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE 2024-12-10T00:14:39,885 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:39,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE because future has completed 2024-12-10T00:14:39,888 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:14:39,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:40,041 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,041 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:14:40,041 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing ce74af60a20cd73f95a0f2738491f1a3, disabling compactions & flushes 2024-12-10T00:14:40,041 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,041 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,041 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. after waiting 0 ms 2024-12-10T00:14:40,041 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,046 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T00:14:40,047 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,047 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for ce74af60a20cd73f95a0f2738491f1a3: Waiting for close lock at 1733789680041Running coprocessor pre-close hooks at 1733789680041Disabling compacts and flushes for region at 1733789680041Disabling writes for close at 1733789680041Writing region close event to WAL at 1733789680042 (+1 ms)Running coprocessor post-close hooks at 1733789680047 (+5 ms)Closed at 1733789680047 2024-12-10T00:14:40,047 WARN [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: ce74af60a20cd73f95a0f2738491f1a3 to self. 2024-12-10T00:14:40,050 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,051 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=CLOSED 2024-12-10T00:14:40,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:14:40,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-10T00:14:40,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 in 166 msec 2024-12-10T00:14:40,057 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE; state=CLOSED, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=true 2024-12-10T00:14:40,208 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE because future has completed 2024-12-10T00:14:40,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:14:40,366 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,366 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:14:40,366 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,366 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:14:40,366 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,366 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,368 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,369 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:40,373 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName A 2024-12-10T00:14:40,375 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:40,375 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:40,376 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,376 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:40,377 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName B 2024-12-10T00:14:40,377 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:40,377 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:40,377 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,378 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:14:40,378 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce74af60a20cd73f95a0f2738491f1a3 columnFamilyName C 2024-12-10T00:14:40,378 DEBUG [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:40,378 INFO [StoreOpener-ce74af60a20cd73f95a0f2738491f1a3-1 {}] regionserver.HStore(327): Store=ce74af60a20cd73f95a0f2738491f1a3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:14:40,379 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,379 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,379 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,380 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,382 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,382 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,382 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:14:40,384 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,385 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened ce74af60a20cd73f95a0f2738491f1a3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64073122, jitterRate=-0.04523608088493347}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:14:40,385 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,386 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for ce74af60a20cd73f95a0f2738491f1a3: Running coprocessor pre-open hook at 1733789680366Writing region info on filesystem at 1733789680366Initializing all the Stores at 1733789680367 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789680367Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789680367Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789680367Cleaning up temporary data from old regions at 1733789680382 (+15 ms)Running coprocessor post-open hooks at 1733789680385 (+3 ms)Region opened successfully at 1733789680386 (+1 ms) 2024-12-10T00:14:40,386 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., pid=39, masterSystemTime=1733789680363 2024-12-10T00:14:40,388 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,388 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=OPEN, openSeqNum=5, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:14:40,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-10T00:14:40,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 in 181 msec 2024-12-10T00:14:40,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-10T00:14:40,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, REOPEN/MOVE in 511 msec 2024-12-10T00:14:40,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-10T00:14:40,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-12-10T00:14:40,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 550 msec 2024-12-10T00:14:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-10T00:14:40,404 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:40,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5739e6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1da99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,410 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ea235c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,411 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70db36b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4770af9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ffc2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da696e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec22e93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,421 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396ac6a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,423 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f6ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca043af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9dacb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@415f5896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62750578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,432 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,434 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc56be6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:14:40,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:14:40,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:14:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:14:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:14:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9c3a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:14:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:14:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:14:40,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:14:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3177e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:14:40,439 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,440 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,440 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,441 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,441 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,441 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,441 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,442 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,443 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:14:40,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,444 DEBUG [Thread-670 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,444 DEBUG [Thread-670 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:14:40,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:14:40,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:40,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-10T00:14:40,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:14:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,449 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,450 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:40,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:14:40,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:14:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:40,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:40,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072df177e2c254df7bd63a6bc1d33db8d_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789680466/Put/seqid=0 2024-12-10T00:14:40,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741957_1133 (size=12154) 2024-12-10T00:14:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:40,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789680487, max timestamp=1733789680560Num uniques 859; 2024-12-10T00:14:40,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:40,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789680487, max timestamp=1733789680560Num uniques 859; 2024-12-10T00:14:40,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:40,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789680487, max timestamp=1733789680560Num uniques 859; 2024-12-10T00:14:40,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:40,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789740593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789740596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789740597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789740598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T00:14:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789740599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789740711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789740720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T00:14:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:40,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T00:14:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:40,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:40,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789740921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789740921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789740921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789740921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789740934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:40,948 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072df177e2c254df7bd63a6bc1d33db8d_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072df177e2c254df7bd63a6bc1d33db8d_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:40,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/051102548fbb4d4b8e534817ea159e4f, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:40,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/051102548fbb4d4b8e534817ea159e4f is 175, key is test_row_0/A:col10/1733789680466/Put/seqid=0 2024-12-10T00:14:40,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741958_1134 (size=30955) 2024-12-10T00:14:40,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/051102548fbb4d4b8e534817ea159e4f 2024-12-10T00:14:41,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/1835025b9a084e36afdc10b2524146b4 is 50, key is test_row_0/B:col10/1733789680466/Put/seqid=0 2024-12-10T00:14:41,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T00:14:41,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:41,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:41,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:41,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:41,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741959_1135 (size=12001) 2024-12-10T00:14:41,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/1835025b9a084e36afdc10b2524146b4 2024-12-10T00:14:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:41,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3eb1c8ed713742d39485f5465d1f061f is 50, key is test_row_0/C:col10/1733789680466/Put/seqid=0 2024-12-10T00:14:41,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741960_1136 (size=12001) 2024-12-10T00:14:41,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3eb1c8ed713742d39485f5465d1f061f 2024-12-10T00:14:41,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/051102548fbb4d4b8e534817ea159e4f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f 2024-12-10T00:14:41,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f, entries=150, sequenceid=15, filesize=30.2 K 2024-12-10T00:14:41,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/1835025b9a084e36afdc10b2524146b4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4 2024-12-10T00:14:41,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T00:14:41,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3eb1c8ed713742d39485f5465d1f061f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f 2024-12-10T00:14:41,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T00:14:41,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ce74af60a20cd73f95a0f2738491f1a3 in 752ms, sequenceid=15, compaction requested=false 2024-12-10T00:14:41,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:41,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T00:14:41,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:41,232 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-10T00:14:41,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:41,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:41,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109378d66c12a341d59b6c444b9c3af18f_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789680596/Put/seqid=0 2024-12-10T00:14:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789741260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789741262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789741263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789741264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789741264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741961_1137 (size=14594) 2024-12-10T00:14:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789741369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789741372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789741380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789741380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789741380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789741579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789741590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789741590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789741590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789741590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:41,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:41,712 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109378d66c12a341d59b6c444b9c3af18f_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109378d66c12a341d59b6c444b9c3af18f_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1fe65c96096f4330b5dba54e55260fb2, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1fe65c96096f4330b5dba54e55260fb2 is 175, key is test_row_0/A:col10/1733789680596/Put/seqid=0 2024-12-10T00:14:41,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741962_1138 (size=39549) 2024-12-10T00:14:41,736 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1fe65c96096f4330b5dba54e55260fb2 2024-12-10T00:14:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/d5d64d6525f649e6ae90ddfb5a587777 is 50, key is test_row_0/B:col10/1733789680596/Put/seqid=0 2024-12-10T00:14:41,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741963_1139 (size=12001) 2024-12-10T00:14:41,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/d5d64d6525f649e6ae90ddfb5a587777 2024-12-10T00:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d18c554d5c534d03a06a63cef855d1e7 is 50, key is test_row_0/C:col10/1733789680596/Put/seqid=0 2024-12-10T00:14:41,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741964_1140 (size=12001) 2024-12-10T00:14:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789741890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789741900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789741900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789741900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789741899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:41,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d18c554d5c534d03a06a63cef855d1e7 2024-12-10T00:14:42,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1fe65c96096f4330b5dba54e55260fb2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2 2024-12-10T00:14:42,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2, entries=200, sequenceid=50, filesize=38.6 K 2024-12-10T00:14:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/d5d64d6525f649e6ae90ddfb5a587777 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777 2024-12-10T00:14:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,314 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:14:42,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d18c554d5c534d03a06a63cef855d1e7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7 2024-12-10T00:14:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,321 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:14:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,323 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for ce74af60a20cd73f95a0f2738491f1a3 in 1090ms, sequenceid=50, compaction requested=false 2024-12-10T00:14:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-10T00:14:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-10T00:14:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-10T00:14:42,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-12-10T00:14:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.8830 sec 2024-12-10T00:14:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:42,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T00:14:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210926af22a34514b5da885ed9c4a8c310c_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,481 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:14:42,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741965_1141 (size=12154) 2024-12-10T00:14:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789742479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789742479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789742483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789742485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789742493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789742600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789742600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789742600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T00:14:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789742610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,611 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789742611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,611 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-10T00:14:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:42,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:42,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:42,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:42,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:42,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:42,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:42,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:42,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789742809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789742809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789742810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789742820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789742820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:42,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:42,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210926af22a34514b5da885ed9c4a8c310c_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210926af22a34514b5da885ed9c4a8c310c_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:42,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/d3ef58ed7ecf4b8bba946eb84d87ac95, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:42,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/d3ef58ed7ecf4b8bba946eb84d87ac95 is 175, key is test_row_0/A:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:42,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741966_1142 (size=30951) 2024-12-10T00:14:42,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/d3ef58ed7ecf4b8bba946eb84d87ac95 2024-12-10T00:14:42,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a56c9dbac69a4c7b88be89814552adb0 is 50, key is test_row_0/B:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:42,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:42,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:42,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:42,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:42,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:42,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741967_1143 (size=9657) 2024-12-10T00:14:43,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789743120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789743120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789743120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789743130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789743130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:43,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:43,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:43,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a56c9dbac69a4c7b88be89814552adb0 2024-12-10T00:14:43,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/4cb3a5478ec9452d936439b0537b1d86 is 50, key is test_row_0/C:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:43,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741968_1144 (size=9657) 2024-12-10T00:14:43,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/4cb3a5478ec9452d936439b0537b1d86 2024-12-10T00:14:43,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/d3ef58ed7ecf4b8bba946eb84d87ac95 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95 2024-12-10T00:14:43,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95, entries=150, sequenceid=62, filesize=30.2 K 2024-12-10T00:14:43,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a56c9dbac69a4c7b88be89814552adb0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0 2024-12-10T00:14:43,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:43,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0, entries=100, sequenceid=62, filesize=9.4 K 2024-12-10T00:14:43,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/4cb3a5478ec9452d936439b0537b1d86 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86 2024-12-10T00:14:43,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86, entries=100, sequenceid=62, filesize=9.4 K 2024-12-10T00:14:43,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ce74af60a20cd73f95a0f2738491f1a3 in 1023ms, sequenceid=62, compaction requested=true 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:43,433 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:43,433 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:43,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:43,436 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:43,436 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:43,436 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:43,436 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:43,436 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,436 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,437 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=32.9 K 2024-12-10T00:14:43,437 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=99.1 K 2024-12-10T00:14:43,437 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,437 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95] 2024-12-10T00:14:43,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 051102548fbb4d4b8e534817ea159e4f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789680466 2024-12-10T00:14:43,438 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1835025b9a084e36afdc10b2524146b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789680466 2024-12-10T00:14:43,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe65c96096f4330b5dba54e55260fb2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789680576 2024-12-10T00:14:43,439 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d3ef58ed7ecf4b8bba946eb84d87ac95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789681261 2024-12-10T00:14:43,439 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5d64d6525f649e6ae90ddfb5a587777, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789680596 2024-12-10T00:14:43,440 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56c9dbac69a4c7b88be89814552adb0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789681261 2024-12-10T00:14:43,465 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:43,472 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:43,472 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/e378439b137f4272be29d255d6a410d9 is 50, key is test_row_0/B:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:43,476 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a166d9b62cb04c6f97dcdf3d61dc195c_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:43,479 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a166d9b62cb04c6f97dcdf3d61dc195c_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:43,479 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a166d9b62cb04c6f97dcdf3d61dc195c_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:43,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741969_1145 (size=12104) 2024-12-10T00:14:43,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741970_1146 (size=4469) 2024-12-10T00:14:43,559 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#126 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:43,560 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/83a47f44110449b694720645ad4e2a61 is 175, key is test_row_0/A:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:43,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,564 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:43,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:43,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741971_1147 (size=31165) 2024-12-10T00:14:43,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210db94dee420d4451c87e8a82988254c00_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789682480/Put/seqid=0 2024-12-10T00:14:43,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:43,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741972_1148 (size=12154) 2024-12-10T00:14:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:43,674 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210db94dee420d4451c87e8a82988254c00_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db94dee420d4451c87e8a82988254c00_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e0587a9c998f4fa6bb6d92a825b7f0e4, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:43,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e0587a9c998f4fa6bb6d92a825b7f0e4 is 175, key is test_row_0/A:col10/1733789682480/Put/seqid=0 2024-12-10T00:14:43,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789743664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741973_1149 (size=30955) 2024-12-10T00:14:43,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789743664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789743677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789743680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789743680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,691 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e0587a9c998f4fa6bb6d92a825b7f0e4 2024-12-10T00:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/e9794a6e7a6f4f689a2c6500c18766d2 is 50, key is test_row_0/B:col10/1733789682480/Put/seqid=0 2024-12-10T00:14:43,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741974_1150 (size=12001) 2024-12-10T00:14:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:43,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789743789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789743800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789743800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789743800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789743800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:43,938 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/e378439b137f4272be29d255d6a410d9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e378439b137f4272be29d255d6a410d9 2024-12-10T00:14:43,947 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into e378439b137f4272be29d255d6a410d9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:43,948 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:43,948 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=13, startTime=1733789683433; duration=0sec 2024-12-10T00:14:43,948 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:43,948 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:43,948 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:43,950 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:43,951 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:43,951 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:43,951 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=32.9 K 2024-12-10T00:14:43,951 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb1c8ed713742d39485f5465d1f061f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789680466 2024-12-10T00:14:43,952 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18c554d5c534d03a06a63cef855d1e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789680596 2024-12-10T00:14:43,952 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb3a5478ec9452d936439b0537b1d86, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789681261 2024-12-10T00:14:43,975 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:43,975 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/23d6adf5e40f472bad66405b0d732c88 is 50, key is test_row_0/C:col10/1733789682404/Put/seqid=0 2024-12-10T00:14:44,006 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/83a47f44110449b694720645ad4e2a61 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61 2024-12-10T00:14:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789744005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,014 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 83a47f44110449b694720645ad4e2a61(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:44,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:44,014 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=13, startTime=1733789683433; duration=0sec 2024-12-10T00:14:44,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:44,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:44,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789744010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789744010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789744010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789744010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741975_1151 (size=12104) 2024-12-10T00:14:44,052 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/23d6adf5e40f472bad66405b0d732c88 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/23d6adf5e40f472bad66405b0d732c88 2024-12-10T00:14:44,062 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into 23d6adf5e40f472bad66405b0d732c88(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:44,062 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:44,062 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=13, startTime=1733789683433; duration=0sec 2024-12-10T00:14:44,062 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:44,062 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:44,129 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/e9794a6e7a6f4f689a2c6500c18766d2 2024-12-10T00:14:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/0bc94dd03c254d22a9277ce9df3ea115 is 50, key is test_row_0/C:col10/1733789682480/Put/seqid=0 2024-12-10T00:14:44,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741976_1152 (size=12001) 2024-12-10T00:14:44,164 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/0bc94dd03c254d22a9277ce9df3ea115 2024-12-10T00:14:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e0587a9c998f4fa6bb6d92a825b7f0e4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4 2024-12-10T00:14:44,186 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4, entries=150, sequenceid=82, filesize=30.2 K 2024-12-10T00:14:44,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/e9794a6e7a6f4f689a2c6500c18766d2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2 2024-12-10T00:14:44,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2, entries=150, sequenceid=82, filesize=11.7 K 2024-12-10T00:14:44,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/0bc94dd03c254d22a9277ce9df3ea115 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115 2024-12-10T00:14:44,221 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115, entries=150, sequenceid=82, filesize=11.7 K 2024-12-10T00:14:44,222 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for ce74af60a20cd73f95a0f2738491f1a3 in 658ms, sequenceid=82, compaction requested=false 2024-12-10T00:14:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-10T00:14:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-10T00:14:44,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T00:14:44,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-12-10T00:14:44,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6160 sec 2024-12-10T00:14:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:44,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T00:14:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:44,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ccfe5aeeab5848a587f1f48d21e64bc1_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:44,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789744340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789744343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789744344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789744345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789744346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741977_1153 (size=12154) 2024-12-10T00:14:44,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:44,369 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ccfe5aeeab5848a587f1f48d21e64bc1_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccfe5aeeab5848a587f1f48d21e64bc1_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:44,370 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1500a7ee27264a109cb84da23f534215, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:44,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1500a7ee27264a109cb84da23f534215 is 175, key is test_row_0/A:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:44,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741978_1154 (size=30955) 2024-12-10T00:14:44,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1500a7ee27264a109cb84da23f534215 2024-12-10T00:14:44,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/ed08665c9597417695854f0c01ff44d1 is 50, key is test_row_0/B:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:44,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741979_1155 (size=12001) 2024-12-10T00:14:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789744450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789744660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789744669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789744669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789744670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789744670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T00:14:44,769 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:44,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-10T00:14:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:44,771 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:44,772 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:44,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:44,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/ed08665c9597417695854f0c01ff44d1 2024-12-10T00:14:44,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/65d00a72343248fcb2c803ce73234d63 is 50, key is test_row_0/C:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741980_1156 (size=12001) 2024-12-10T00:14:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:44,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T00:14:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:44,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:44,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:44,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:44,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789744970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789744979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789744979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789744979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789744979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T00:14:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:45,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T00:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:45,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/65d00a72343248fcb2c803ce73234d63 2024-12-10T00:14:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/1500a7ee27264a109cb84da23f534215 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215 2024-12-10T00:14:45,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215, entries=150, sequenceid=101, filesize=30.2 K 2024-12-10T00:14:45,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/ed08665c9597417695854f0c01ff44d1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1 2024-12-10T00:14:45,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1, entries=150, sequenceid=101, filesize=11.7 K 2024-12-10T00:14:45,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/65d00a72343248fcb2c803ce73234d63 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63 2024-12-10T00:14:45,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63, entries=150, sequenceid=101, filesize=11.7 K 2024-12-10T00:14:45,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ce74af60a20cd73f95a0f2738491f1a3 in 975ms, sequenceid=101, compaction requested=true 2024-12-10T00:14:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:45,298 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:45,298 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:45,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:45,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:45,302 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:45,303 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:45,303 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,303 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=90.9 K 2024-12-10T00:14:45,303 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,303 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215] 2024-12-10T00:14:45,303 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:45,303 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:45,303 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,303 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e378439b137f4272be29d255d6a410d9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=35.3 K 2024-12-10T00:14:45,303 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a47f44110449b694720645ad4e2a61, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789680597 2024-12-10T00:14:45,304 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e378439b137f4272be29d255d6a410d9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789680597 2024-12-10T00:14:45,304 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0587a9c998f4fa6bb6d92a825b7f0e4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733789682478 2024-12-10T00:14:45,304 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e9794a6e7a6f4f689a2c6500c18766d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733789682478 2024-12-10T00:14:45,304 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1500a7ee27264a109cb84da23f534215, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:45,305 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ed08665c9597417695854f0c01ff44d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:45,314 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:45,315 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/071bedc24eec416f8a782c0a5cf0afa5 is 50, key is test_row_0/B:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:45,327 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:45,332 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104dd9c1d7985e43fca33d296ea0209000_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:45,335 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104dd9c1d7985e43fca33d296ea0209000_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:45,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741981_1157 (size=12207) 2024-12-10T00:14:45,335 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104dd9c1d7985e43fca33d296ea0209000_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:45,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741982_1158 (size=4469) 2024-12-10T00:14:45,362 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#136 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:45,362 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/005b47d3a4f340afa5d7989e32ef741c is 175, key is test_row_0/A:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:45,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T00:14:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T00:14:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:45,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:45,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741983_1159 (size=31161) 2024-12-10T00:14:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108a9c6290168e460f8dcc001aac63c8a2_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789684345/Put/seqid=0 2024-12-10T00:14:45,397 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/005b47d3a4f340afa5d7989e32ef741c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c 2024-12-10T00:14:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:45,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741984_1160 (size=12154) 2024-12-10T00:14:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:45,405 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 005b47d3a4f340afa5d7989e32ef741c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:45,405 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:45,406 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=13, startTime=1733789685298; duration=0sec 2024-12-10T00:14:45,406 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:45,406 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:45,406 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:45,411 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:45,411 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:45,412 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:45,412 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/23d6adf5e40f472bad66405b0d732c88, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=35.3 K 2024-12-10T00:14:45,417 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d6adf5e40f472bad66405b0d732c88, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789680597 2024-12-10T00:14:45,418 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc94dd03c254d22a9277ce9df3ea115, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733789682478 2024-12-10T00:14:45,418 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d00a72343248fcb2c803ce73234d63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:45,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108a9c6290168e460f8dcc001aac63c8a2_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108a9c6290168e460f8dcc001aac63c8a2_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/da3844ce305a46c4a4265bed14c20679, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/da3844ce305a46c4a4265bed14c20679 is 175, key is test_row_0/A:col10/1733789684345/Put/seqid=0 2024-12-10T00:14:45,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741985_1161 (size=30955) 2024-12-10T00:14:45,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/da3844ce305a46c4a4265bed14c20679 2024-12-10T00:14:45,447 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:45,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/c55cf36884b24bbb8cd211c7c95b2bbe is 50, key is test_row_0/B:col10/1733789684345/Put/seqid=0 2024-12-10T00:14:45,447 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/0e71528178ca4a77be645c57c22743e5 is 50, key is test_row_0/C:col10/1733789683671/Put/seqid=0 2024-12-10T00:14:45,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:45,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741987_1163 (size=12001) 2024-12-10T00:14:45,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741986_1162 (size=12207) 2024-12-10T00:14:45,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789745565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,577 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/0e71528178ca4a77be645c57c22743e5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0e71528178ca4a77be645c57c22743e5 2024-12-10T00:14:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789745566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789745569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789745571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789745579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,590 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into 0e71528178ca4a77be645c57c22743e5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:45,590 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:45,590 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=13, startTime=1733789685298; duration=0sec 2024-12-10T00:14:45,590 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:45,590 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789745690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789745690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789745690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789745691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789745691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,753 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/071bedc24eec416f8a782c0a5cf0afa5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/071bedc24eec416f8a782c0a5cf0afa5 2024-12-10T00:14:45,760 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into 071bedc24eec416f8a782c0a5cf0afa5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:45,760 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:45,760 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=13, startTime=1733789685298; duration=0sec 2024-12-10T00:14:45,761 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:45,761 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:45,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789745900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789745901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789745900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789745901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:45,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789745910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:45,962 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/c55cf36884b24bbb8cd211c7c95b2bbe 2024-12-10T00:14:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fc4e5bc7f642478c99666aaeb6d122fa is 50, key is test_row_0/C:col10/1733789684345/Put/seqid=0 2024-12-10T00:14:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741988_1164 (size=12001) 2024-12-10T00:14:46,025 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fc4e5bc7f642478c99666aaeb6d122fa 2024-12-10T00:14:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/da3844ce305a46c4a4265bed14c20679 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679 2024-12-10T00:14:46,045 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679, entries=150, sequenceid=117, filesize=30.2 K 2024-12-10T00:14:46,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/c55cf36884b24bbb8cd211c7c95b2bbe as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe 2024-12-10T00:14:46,056 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T00:14:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fc4e5bc7f642478c99666aaeb6d122fa as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa 2024-12-10T00:14:46,067 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa, entries=150, sequenceid=117, filesize=11.7 K 2024-12-10T00:14:46,068 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ce74af60a20cd73f95a0f2738491f1a3 in 686ms, sequenceid=117, compaction requested=false 2024-12-10T00:14:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-10T00:14:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-10T00:14:46,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T00:14:46,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3000 sec 2024-12-10T00:14:46,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3040 sec 2024-12-10T00:14:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:46,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:46,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f27ef003ef0e494b9b37da22f719317d_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:46,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741989_1165 (size=22218) 2024-12-10T00:14:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789746247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789746251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789746253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789746257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789746267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f27ef003ef0e494b9b37da22f719317d_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27ef003ef0e494b9b37da22f719317d_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:46,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f2cc1de5bfae4526a6d613f56d80713d, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:46,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f2cc1de5bfae4526a6d613f56d80713d is 175, key is test_row_0/A:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741990_1166 (size=65623) 2024-12-10T00:14:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789746369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789746380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789746380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789746380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789746380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789746580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789746590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789746590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789746590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789746590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f2cc1de5bfae4526a6d613f56d80713d 2024-12-10T00:14:46,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/6382c859135d459d88d956c23525e828 is 50, key is test_row_0/B:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:46,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741991_1167 (size=12101) 2024-12-10T00:14:46,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789746890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789746909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789746909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789746909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789746909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T00:14:46,920 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:46,920 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-10T00:14:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:46,922 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:46,922 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:46,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:47,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T00:14:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/6382c859135d459d88d956c23525e828 2024-12-10T00:14:47,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/9b455942b31b4d43b25f1a390ee89ce8 is 50, key is test_row_0/C:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:47,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741992_1168 (size=12101) 2024-12-10T00:14:47,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/9b455942b31b4d43b25f1a390ee89ce8 2024-12-10T00:14:47,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f2cc1de5bfae4526a6d613f56d80713d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d 2024-12-10T00:14:47,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d, entries=350, sequenceid=140, filesize=64.1 K 2024-12-10T00:14:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/6382c859135d459d88d956c23525e828 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828 2024-12-10T00:14:47,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828, entries=150, sequenceid=140, filesize=11.8 K 2024-12-10T00:14:47,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/9b455942b31b4d43b25f1a390ee89ce8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8 2024-12-10T00:14:47,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T00:14:47,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:47,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:47,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8, entries=150, sequenceid=140, filesize=11.8 K 2024-12-10T00:14:47,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ce74af60a20cd73f95a0f2738491f1a3 in 1016ms, sequenceid=140, compaction requested=true 2024-12-10T00:14:47,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:47,236 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:47,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:47,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:47,236 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:47,237 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:47,237 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:47,237 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:47,237 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:47,237 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,238 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,238 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/071bedc24eec416f8a782c0a5cf0afa5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=35.5 K 2024-12-10T00:14:47,238 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=124.7 K 2024-12-10T00:14:47,238 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,238 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d] 2024-12-10T00:14:47,238 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 071bedc24eec416f8a782c0a5cf0afa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:47,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:47,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:47,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:47,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005b47d3a4f340afa5d7989e32ef741c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:47,239 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c55cf36884b24bbb8cd211c7c95b2bbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733789684338 2024-12-10T00:14:47,239 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting da3844ce305a46c4a4265bed14c20679, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733789684338 2024-12-10T00:14:47,239 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6382c859135d459d88d956c23525e828, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:47,241 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2cc1de5bfae4526a6d613f56d80713d, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:47,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:47,260 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:47,262 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/084c91d7db104148a6ad96c9f88b490e is 50, key is test_row_0/B:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:47,265 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:47,281 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a2bc5f2b10fa488da3d7f3b8e3919864_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:47,283 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a2bc5f2b10fa488da3d7f3b8e3919864_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:47,283 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a2bc5f2b10fa488da3d7f3b8e3919864_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741993_1169 (size=12409) 2024-12-10T00:14:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741994_1170 (size=4469) 2024-12-10T00:14:47,323 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#145 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:47,323 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0aff8d584c9b4c42ad0d915d75642d32 is 175, key is test_row_0/A:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:47,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741995_1171 (size=31363) 2024-12-10T00:14:47,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T00:14:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,389 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T00:14:47,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:47,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121064b905a69541424ca85984121f7209bb_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789686262/Put/seqid=0 2024-12-10T00:14:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:47,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741996_1172 (size=12304) 2024-12-10T00:14:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789747445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789747447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789747449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789747451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789747453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:47,476 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121064b905a69541424ca85984121f7209bb_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064b905a69541424ca85984121f7209bb_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8187b0a979234006a981e636e38708a6, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8187b0a979234006a981e636e38708a6 is 175, key is test_row_0/A:col10/1733789686262/Put/seqid=0 2024-12-10T00:14:47,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741997_1173 (size=31105) 2024-12-10T00:14:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789747559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789747559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789747559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789747569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789747570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,707 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/084c91d7db104148a6ad96c9f88b490e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/084c91d7db104148a6ad96c9f88b490e 2024-12-10T00:14:47,712 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into 084c91d7db104148a6ad96c9f88b490e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:47,712 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:47,712 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=13, startTime=1733789687236; duration=0sec 2024-12-10T00:14:47,712 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:47,712 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:47,712 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:47,713 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:47,714 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:47,714 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:47,714 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0e71528178ca4a77be645c57c22743e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=35.5 K 2024-12-10T00:14:47,714 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e71528178ca4a77be645c57c22743e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789683671 2024-12-10T00:14:47,715 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4e5bc7f642478c99666aaeb6d122fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733789684338 2024-12-10T00:14:47,715 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b455942b31b4d43b25f1a390ee89ce8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:47,727 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:47,727 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/cf62bf9ef1274e14812c74a7102ef2ee is 50, key is test_row_0/C:col10/1733789685496/Put/seqid=0 2024-12-10T00:14:47,738 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0aff8d584c9b4c42ad0d915d75642d32 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32 2024-12-10T00:14:47,744 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 0aff8d584c9b4c42ad0d915d75642d32(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:47,744 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:47,744 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=13, startTime=1733789687235; duration=0sec 2024-12-10T00:14:47,745 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:47,745 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:47,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741998_1174 (size=12409) 2024-12-10T00:14:47,752 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/cf62bf9ef1274e14812c74a7102ef2ee as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/cf62bf9ef1274e14812c74a7102ef2ee 2024-12-10T00:14:47,760 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into cf62bf9ef1274e14812c74a7102ef2ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:47,760 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:47,760 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=13, startTime=1733789687239; duration=0sec 2024-12-10T00:14:47,761 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:47,761 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789747769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789747769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789747769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789747780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789747780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:47,891 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8187b0a979234006a981e636e38708a6 2024-12-10T00:14:47,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b466997cc22c4818a5242add1c741b97 is 50, key is test_row_0/B:col10/1733789686262/Put/seqid=0 2024-12-10T00:14:47,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741999_1175 (size=12151) 2024-12-10T00:14:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789748079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789748079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789748079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789748090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789748090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,307 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b466997cc22c4818a5242add1c741b97 2024-12-10T00:14:48,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d3245bbd84514dd39d18e36e16e12f04 is 50, key is test_row_0/C:col10/1733789686262/Put/seqid=0 2024-12-10T00:14:48,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742000_1176 (size=12151) 2024-12-10T00:14:48,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789748590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789748593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789748593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789748599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789748599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:48,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:48,757 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d3245bbd84514dd39d18e36e16e12f04 2024-12-10T00:14:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8187b0a979234006a981e636e38708a6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6 2024-12-10T00:14:48,803 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6, entries=150, sequenceid=152, filesize=30.4 K 2024-12-10T00:14:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b466997cc22c4818a5242add1c741b97 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97 2024-12-10T00:14:48,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97, entries=150, sequenceid=152, filesize=11.9 K 2024-12-10T00:14:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/d3245bbd84514dd39d18e36e16e12f04 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04 2024-12-10T00:14:48,844 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04, entries=150, sequenceid=152, filesize=11.9 K 2024-12-10T00:14:48,845 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ce74af60a20cd73f95a0f2738491f1a3 in 1456ms, sequenceid=152, compaction requested=false 2024-12-10T00:14:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-10T00:14:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-10T00:14:48,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T00:14:48,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-12-10T00:14:48,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9290 sec 2024-12-10T00:14:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T00:14:49,069 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:49,070 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-10T00:14:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:49,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:49,073 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:49,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:49,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T00:14:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:49,226 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:49,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:49,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:49,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104faa9e5bda92411e8ae0b52ddfaeac85_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789687447/Put/seqid=0 2024-12-10T00:14:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742001_1177 (size=12304) 2024-12-10T00:14:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:49,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789749618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789749618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789749620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789749620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789749620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,629 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,633 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:49,661 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104faa9e5bda92411e8ae0b52ddfaeac85_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104faa9e5bda92411e8ae0b52ddfaeac85_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:49,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/cc811250372148879e83a5b810e34f21, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:49,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/cc811250372148879e83a5b810e34f21 is 175, key is test_row_0/A:col10/1733789687447/Put/seqid=0 2024-12-10T00:14:49,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742002_1178 (size=31105) 2024-12-10T00:14:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789749739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789749739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789749739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,805 INFO [master/a67c4886b4f7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T00:14:49,805 INFO [master/a67c4886b4f7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T00:14:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789749949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789749949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789749949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,078 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/cc811250372148879e83a5b810e34f21 2024-12-10T00:14:50,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/eeffa6151e1048c0839e9277ebeae3ab is 50, key is test_row_0/B:col10/1733789687447/Put/seqid=0 2024-12-10T00:14:50,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742003_1179 (size=12151) 2024-12-10T00:14:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:50,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789750260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789750260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789750260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,503 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/eeffa6151e1048c0839e9277ebeae3ab 2024-12-10T00:14:50,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/ef2a6287386b4f07864e24220c51102d is 50, key is test_row_0/C:col10/1733789687447/Put/seqid=0 2024-12-10T00:14:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742004_1180 (size=12151) 2024-12-10T00:14:50,550 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/ef2a6287386b4f07864e24220c51102d 2024-12-10T00:14:50,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/cc811250372148879e83a5b810e34f21 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21 2024-12-10T00:14:50,562 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21, entries=150, sequenceid=175, filesize=30.4 K 2024-12-10T00:14:50,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/eeffa6151e1048c0839e9277ebeae3ab as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab 2024-12-10T00:14:50,570 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T00:14:50,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/ef2a6287386b4f07864e24220c51102d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d 2024-12-10T00:14:50,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d, entries=150, sequenceid=175, filesize=11.9 K 2024-12-10T00:14:50,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ce74af60a20cd73f95a0f2738491f1a3 in 1352ms, sequenceid=175, compaction requested=true 2024-12-10T00:14:50,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:50,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:50,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-10T00:14:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-10T00:14:50,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-10T00:14:50,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-12-10T00:14:50,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5140 sec 2024-12-10T00:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:50,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:50,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:50,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102fa54ebf2769425ca093e7447d0f0106_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:50,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742005_1181 (size=14794) 2024-12-10T00:14:50,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789750799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789750800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789750800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789750909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789750919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789750919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789751120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789751129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789751129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:51,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102fa54ebf2769425ca093e7447d0f0106_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102fa54ebf2769425ca093e7447d0f0106_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:51,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f496ed5e8c2c453e88467655e7b790cb, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:51,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f496ed5e8c2c453e88467655e7b790cb is 175, key is test_row_0/A:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:51,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742006_1182 (size=39749) 2024-12-10T00:14:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T00:14:51,219 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:51,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-10T00:14:51,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:51,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:51,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:51,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789751430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789751440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789751440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:51,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:51,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f496ed5e8c2c453e88467655e7b790cb 2024-12-10T00:14:51,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/fcdb8a1884f6435d9f3b869a1e1b7e7f is 50, key is test_row_0/B:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:51,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742007_1183 (size=12151) 2024-12-10T00:14:51,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789751640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,642 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789751650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,654 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:51,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:51,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789751939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789751950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789751950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:51,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:51,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/fcdb8a1884f6435d9f3b869a1e1b7e7f 2024-12-10T00:14:52,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/7ed104fd865548c2b0dba190e82f12bb is 50, key is test_row_0/C:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:52,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742008_1184 (size=12151) 2024-12-10T00:14:52,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:52,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:52,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/7ed104fd865548c2b0dba190e82f12bb 2024-12-10T00:14:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/f496ed5e8c2c453e88467655e7b790cb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb 2024-12-10T00:14:52,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb, entries=200, sequenceid=188, filesize=38.8 K 2024-12-10T00:14:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/fcdb8a1884f6435d9f3b869a1e1b7e7f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f 2024-12-10T00:14:52,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f, entries=150, sequenceid=188, filesize=11.9 K 2024-12-10T00:14:52,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/7ed104fd865548c2b0dba190e82f12bb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb 2024-12-10T00:14:52,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb, entries=150, sequenceid=188, filesize=11.9 K 2024-12-10T00:14:52,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ce74af60a20cd73f95a0f2738491f1a3 in 1702ms, sequenceid=188, compaction requested=true 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:52,475 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:52,475 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:14:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:52,477 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:52,477 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,477 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=130.2 K 2024-12-10T00:14:52,477 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,477 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/084c91d7db104148a6ad96c9f88b490e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=47.7 K 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb] 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aff8d584c9b4c42ad0d915d75642d32, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:52,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 084c91d7db104148a6ad96c9f88b490e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8187b0a979234006a981e636e38708a6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733789686248 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b466997cc22c4818a5242add1c741b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733789686248 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc811250372148879e83a5b810e34f21, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733789687442 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting eeffa6151e1048c0839e9277ebeae3ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733789687442 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f496ed5e8c2c453e88467655e7b790cb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689615 2024-12-10T00:14:52,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fcdb8a1884f6435d9f3b869a1e1b7e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689616 2024-12-10T00:14:52,488 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:52,489 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:52,489 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a86cad77cf5144fcb783201916d258e1 is 50, key is test_row_0/B:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:52,491 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412106f4d1aa1f28c4db49722fda1d5e6dd39_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742009_1185 (size=12595) 2024-12-10T00:14:52,493 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412106f4d1aa1f28c4db49722fda1d5e6dd39_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:52,493 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106f4d1aa1f28c4db49722fda1d5e6dd39_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:52,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742010_1186 (size=4469) 2024-12-10T00:14:52,499 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#157 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:52,500 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/35cb36c4d1d84f6e81755f26cf3d5a8b is 175, key is test_row_0/A:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:52,501 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a86cad77cf5144fcb783201916d258e1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a86cad77cf5144fcb783201916d258e1 2024-12-10T00:14:52,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742011_1187 (size=31549) 2024-12-10T00:14:52,509 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into a86cad77cf5144fcb783201916d258e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:52,509 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:52,509 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=12, startTime=1733789692475; duration=0sec 2024-12-10T00:14:52,509 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:52,509 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:52,510 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:14:52,511 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:14:52,511 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:52,512 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,512 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/cf62bf9ef1274e14812c74a7102ef2ee, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=47.7 K 2024-12-10T00:14:52,512 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cf62bf9ef1274e14812c74a7102ef2ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733789685496 2024-12-10T00:14:52,513 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d3245bbd84514dd39d18e36e16e12f04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733789686248 2024-12-10T00:14:52,514 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2a6287386b4f07864e24220c51102d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733789687442 2024-12-10T00:14:52,514 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed104fd865548c2b0dba190e82f12bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689616 2024-12-10T00:14:52,518 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/35cb36c4d1d84f6e81755f26cf3d5a8b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b 2024-12-10T00:14:52,524 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 35cb36c4d1d84f6e81755f26cf3d5a8b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:52,524 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:52,524 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=12, startTime=1733789692475; duration=0sec 2024-12-10T00:14:52,524 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:52,525 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:52,527 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:52,527 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/35a125c14bc244e28fcdee72fdd1b5bc is 50, key is test_row_0/C:col10/1733789690770/Put/seqid=0 2024-12-10T00:14:52,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742012_1188 (size=12595) 2024-12-10T00:14:52,538 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/35a125c14bc244e28fcdee72fdd1b5bc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/35a125c14bc244e28fcdee72fdd1b5bc 2024-12-10T00:14:52,544 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into 35a125c14bc244e28fcdee72fdd1b5bc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:52,544 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:52,544 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=12, startTime=1733789692475; duration=0sec 2024-12-10T00:14:52,544 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:52,544 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:52,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-10T00:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:52,598 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f30d3838d5704c7a99738c2b15896a04_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789690799/Put/seqid=0 2024-12-10T00:14:52,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742013_1189 (size=12304) 2024-12-10T00:14:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:52,622 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f30d3838d5704c7a99738c2b15896a04_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f30d3838d5704c7a99738c2b15896a04_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8b862238766e4e2983d052501315fad1, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8b862238766e4e2983d052501315fad1 is 175, key is test_row_0/A:col10/1733789690799/Put/seqid=0 2024-12-10T00:14:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742014_1190 (size=31105) 2024-12-10T00:14:52,630 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8b862238766e4e2983d052501315fad1 2024-12-10T00:14:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b2d1f79edb95481388d5a2d26b09b3a5 is 50, key is test_row_0/B:col10/1733789690799/Put/seqid=0 2024-12-10T00:14:52,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742015_1191 (size=12151) 2024-12-10T00:14:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789752972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:52,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789752973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789752974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,047 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b2d1f79edb95481388d5a2d26b09b3a5 2024-12-10T00:14:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/77f1a85fd3d440b886b4ab07aeb10d82 is 50, key is test_row_0/C:col10/1733789690799/Put/seqid=0 2024-12-10T00:14:53,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742016_1192 (size=12151) 2024-12-10T00:14:53,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789753080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789753080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789753080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789753290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789753290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789753290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:53,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/77f1a85fd3d440b886b4ab07aeb10d82 2024-12-10T00:14:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/8b862238766e4e2983d052501315fad1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1 2024-12-10T00:14:53,483 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1, entries=150, sequenceid=210, filesize=30.4 K 2024-12-10T00:14:53,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b2d1f79edb95481388d5a2d26b09b3a5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5 2024-12-10T00:14:53,490 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5, entries=150, sequenceid=210, filesize=11.9 K 2024-12-10T00:14:53,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/77f1a85fd3d440b886b4ab07aeb10d82 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82 2024-12-10T00:14:53,496 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82, entries=150, sequenceid=210, filesize=11.9 K 2024-12-10T00:14:53,497 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ce74af60a20cd73f95a0f2738491f1a3 in 900ms, sequenceid=210, compaction requested=false 2024-12-10T00:14:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-10T00:14:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-10T00:14:53,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-10T00:14:53,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-12-10T00:14:53,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2830 sec 2024-12-10T00:14:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:53,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:53,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:53,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121098b3f611f2bc4b36879750e39c168c19_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:53,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742017_1193 (size=12304) 2024-12-10T00:14:53,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:53,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121098b3f611f2bc4b36879750e39c168c19_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121098b3f611f2bc4b36879750e39c168c19_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:53,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/794ece6daf324b33b37a2053e719b040, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:53,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789753630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/794ece6daf324b33b37a2053e719b040 is 175, key is test_row_0/A:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789753635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789753638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742018_1194 (size=31105) 2024-12-10T00:14:53,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789753740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789753757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789753757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789753949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789753970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:53,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789753973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,059 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/794ece6daf324b33b37a2053e719b040 2024-12-10T00:14:54,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/59ca48ac1d94468e80191e11780fe986 is 50, key is test_row_0/B:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:54,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742019_1195 (size=12151) 2024-12-10T00:14:54,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/59ca48ac1d94468e80191e11780fe986 2024-12-10T00:14:54,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fdfaa53c03424476b802b7cc4c07c81f is 50, key is test_row_0/C:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:54,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742020_1196 (size=12151) 2024-12-10T00:14:54,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789754259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789754280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789754280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fdfaa53c03424476b802b7cc4c07c81f 2024-12-10T00:14:54,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/794ece6daf324b33b37a2053e719b040 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040 2024-12-10T00:14:54,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040, entries=150, sequenceid=224, filesize=30.4 K 2024-12-10T00:14:54,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/59ca48ac1d94468e80191e11780fe986 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986 2024-12-10T00:14:54,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T00:14:54,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fdfaa53c03424476b802b7cc4c07c81f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f 2024-12-10T00:14:54,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T00:14:54,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ce74af60a20cd73f95a0f2738491f1a3 in 977ms, sequenceid=224, compaction requested=true 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:54,579 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:54,579 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:54,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:54,580 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:54,580 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:54,580 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=91.6 K 2024-12-10T00:14:54,580 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a86cad77cf5144fcb783201916d258e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=36.0 K 2024-12-10T00:14:54,580 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040] 2024-12-10T00:14:54,580 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a86cad77cf5144fcb783201916d258e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689616 2024-12-10T00:14:54,581 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35cb36c4d1d84f6e81755f26cf3d5a8b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689616 2024-12-10T00:14:54,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b2d1f79edb95481388d5a2d26b09b3a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733789690784 2024-12-10T00:14:54,581 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b862238766e4e2983d052501315fad1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733789690784 2024-12-10T00:14:54,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ca48ac1d94468e80191e11780fe986, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:54,581 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794ece6daf324b33b37a2053e719b040, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:54,595 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:54,597 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:54,597 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/3f5b7be038d54ed3a4520370e8cd677b is 50, key is test_row_0/B:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742021_1197 (size=12697) 2024-12-10T00:14:54,619 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412100ce6bc3275af46a7bb6b6dfe97be9262_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:54,621 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412100ce6bc3275af46a7bb6b6dfe97be9262_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:54,625 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100ce6bc3275af46a7bb6b6dfe97be9262_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:54,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742022_1198 (size=4469) 2024-12-10T00:14:54,639 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#166 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:54,640 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/4db28f4782c94636ad04bca72a66513e is 175, key is test_row_0/A:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:54,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742023_1199 (size=31651) 2024-12-10T00:14:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:54,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:54,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:54,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121046716fdc11c945af8379a1a95c426ac2_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789694770/Put/seqid=0 2024-12-10T00:14:54,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742024_1200 (size=14794) 2024-12-10T00:14:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789754809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789754813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789754813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789754919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789754919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:54,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789754930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,007 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/3f5b7be038d54ed3a4520370e8cd677b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f5b7be038d54ed3a4520370e8cd677b 2024-12-10T00:14:55,013 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into 3f5b7be038d54ed3a4520370e8cd677b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:55,013 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:55,013 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=13, startTime=1733789694579; duration=0sec 2024-12-10T00:14:55,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:55,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:55,014 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:14:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:14:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:55,017 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,017 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/35a125c14bc244e28fcdee72fdd1b5bc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=36.0 K 2024-12-10T00:14:55,018 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a125c14bc244e28fcdee72fdd1b5bc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733789689616 2024-12-10T00:14:55,018 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f1a85fd3d440b886b4ab07aeb10d82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733789690784 2024-12-10T00:14:55,018 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfaa53c03424476b802b7cc4c07c81f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:55,024 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:55,025 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/2f8848044036446dba16b795a4f35b87 is 50, key is test_row_0/C:col10/1733789693600/Put/seqid=0 2024-12-10T00:14:55,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742025_1201 (size=12697) 2024-12-10T00:14:55,050 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/4db28f4782c94636ad04bca72a66513e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e 2024-12-10T00:14:55,056 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 4db28f4782c94636ad04bca72a66513e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:55,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:55,056 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=13, startTime=1733789694579; duration=0sec 2024-12-10T00:14:55,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:55,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:55,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789755129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789755129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789755140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,208 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:55,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121046716fdc11c945af8379a1a95c426ac2_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121046716fdc11c945af8379a1a95c426ac2_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:55,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0b33b5ada44b4850a302cb95d50117e1, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:55,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0b33b5ada44b4850a302cb95d50117e1 is 175, key is test_row_0/A:col10/1733789694770/Put/seqid=0 2024-12-10T00:14:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742026_1202 (size=39749) 2024-12-10T00:14:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T00:14:55,390 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:55,390 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-10T00:14:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:55,392 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:55,392 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:55,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:55,433 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/2f8848044036446dba16b795a4f35b87 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/2f8848044036446dba16b795a4f35b87 2024-12-10T00:14:55,437 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into 2f8848044036446dba16b795a4f35b87(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:55,437 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:55,437 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=13, startTime=1733789694579; duration=0sec 2024-12-10T00:14:55,437 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:55,437 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789755439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789755439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789755450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:55,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0b33b5ada44b4850a302cb95d50117e1 2024-12-10T00:14:55,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/3f73c572fe5f4d6f81dd7a44c3340792 is 50, key is test_row_0/B:col10/1733789694770/Put/seqid=0 2024-12-10T00:14:55,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742027_1203 (size=12151) 2024-12-10T00:14:55,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59426 deadline: 1733789755669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,672 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59420 deadline: 1733789755690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,691 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in ce74af60a20cd73f95a0f2738491f1a3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:14:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:55,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:55,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:55,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789755950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789755950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789755959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:56,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:56,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/3f73c572fe5f4d6f81dd7a44c3340792 2024-12-10T00:14:56,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3e21e04022a345a39a0a589907fecaf1 is 50, key is test_row_0/C:col10/1733789694770/Put/seqid=0 2024-12-10T00:14:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742028_1204 (size=12151) 2024-12-10T00:14:56,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:56,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3e21e04022a345a39a0a589907fecaf1 2024-12-10T00:14:56,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/0b33b5ada44b4850a302cb95d50117e1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1 2024-12-10T00:14:56,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1, entries=200, sequenceid=244, filesize=38.8 K 2024-12-10T00:14:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/3f73c572fe5f4d6f81dd7a44c3340792 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792 2024-12-10T00:14:56,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T00:14:56,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/3e21e04022a345a39a0a589907fecaf1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1 2024-12-10T00:14:56,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:56,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:56,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1, entries=150, sequenceid=244, filesize=11.9 K 2024-12-10T00:14:56,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ce74af60a20cd73f95a0f2738491f1a3 in 1685ms, sequenceid=244, compaction requested=false 2024-12-10T00:14:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:56,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-10T00:14:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:56,615 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210955761175dc24b18a2b863360a6bf571_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789694805/Put/seqid=0 2024-12-10T00:14:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742029_1205 (size=12304) 2024-12-10T00:14:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:56,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789756980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789756981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789756982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:57,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210955761175dc24b18a2b863360a6bf571_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210955761175dc24b18a2b863360a6bf571_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/fba023bc01f74844b6df0a24c7955e7e, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/fba023bc01f74844b6df0a24c7955e7e is 175, key is test_row_0/A:col10/1733789694805/Put/seqid=0 2024-12-10T00:14:57,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742030_1206 (size=31105) 2024-12-10T00:14:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789757089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789757092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789757092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789757300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789757300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789757300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,447 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/fba023bc01f74844b6df0a24c7955e7e 2024-12-10T00:14:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a220d4cefac349109a4aedcd9784229f is 50, key is test_row_0/B:col10/1733789694805/Put/seqid=0 2024-12-10T00:14:57,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742031_1207 (size=12151) 2024-12-10T00:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:57,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789757609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789757610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789757610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:57,859 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a220d4cefac349109a4aedcd9784229f 2024-12-10T00:14:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/19c42ad088fd45de9f27bb7c5686585b is 50, key is test_row_0/C:col10/1733789694805/Put/seqid=0 2024-12-10T00:14:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742032_1208 (size=12151) 2024-12-10T00:14:57,870 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/19c42ad088fd45de9f27bb7c5686585b 2024-12-10T00:14:57,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/fba023bc01f74844b6df0a24c7955e7e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e 2024-12-10T00:14:57,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e, entries=150, sequenceid=260, filesize=30.4 K 2024-12-10T00:14:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/a220d4cefac349109a4aedcd9784229f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f 2024-12-10T00:14:57,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f, entries=150, sequenceid=260, filesize=11.9 K 2024-12-10T00:14:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/19c42ad088fd45de9f27bb7c5686585b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b 2024-12-10T00:14:57,888 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b, entries=150, sequenceid=260, filesize=11.9 K 2024-12-10T00:14:57,889 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ce74af60a20cd73f95a0f2738491f1a3 in 1275ms, sequenceid=260, compaction requested=true 2024-12-10T00:14:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-10T00:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-10T00:14:57,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-10T00:14:57,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-12-10T00:14:57,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5030 sec 2024-12-10T00:14:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:58,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:58,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b4c6fb57374e42e1945be7cd87eb7154_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:58,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742033_1209 (size=17534) 2024-12-10T00:14:58,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789758137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789758137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b4c6fb57374e42e1945be7cd87eb7154_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b4c6fb57374e42e1945be7cd87eb7154_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:58,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/c1c87c23d266419e9ce72327501a9dce, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:58,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/c1c87c23d266419e9ce72327501a9dce is 175, key is test_row_0/A:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:58,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789758139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742034_1210 (size=48639) 2024-12-10T00:14:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789758250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789758460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789758460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789758460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,662 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/c1c87c23d266419e9ce72327501a9dce 2024-12-10T00:14:58,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b0d839c5735d403a9fd0f4947c5f1a29 is 50, key is test_row_0/B:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:58,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742035_1211 (size=12301) 2024-12-10T00:14:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789758769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789758769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789758769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b0d839c5735d403a9fd0f4947c5f1a29 2024-12-10T00:14:59,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/dc1d902806234d81b12d0d4f7cfc26de is 50, key is test_row_0/C:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:59,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742036_1212 (size=12301) 2024-12-10T00:14:59,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/dc1d902806234d81b12d0d4f7cfc26de 2024-12-10T00:14:59,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/c1c87c23d266419e9ce72327501a9dce as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce 2024-12-10T00:14:59,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce, entries=250, sequenceid=282, filesize=47.5 K 2024-12-10T00:14:59,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b0d839c5735d403a9fd0f4947c5f1a29 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29 2024-12-10T00:14:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29, entries=150, sequenceid=282, filesize=12.0 K 2024-12-10T00:14:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/dc1d902806234d81b12d0d4f7cfc26de as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de 2024-12-10T00:14:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de, entries=150, sequenceid=282, filesize=12.0 K 2024-12-10T00:14:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for ce74af60a20cd73f95a0f2738491f1a3 in 1002ms, sequenceid=282, compaction requested=true 2024-12-10T00:14:59,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:59,124 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:59,125 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:14:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,131 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:14:59,132 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:14:59,132 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,132 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f5b7be038d54ed3a4520370e8cd677b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=48.1 K 2024-12-10T00:14:59,133 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:14:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,133 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:14:59,133 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,133 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=147.6 K 2024-12-10T00:14:59,133 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,133 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce] 2024-12-10T00:14:59,133 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5b7be038d54ed3a4520370e8cd677b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:59,134 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db28f4782c94636ad04bca72a66513e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:59,134 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f73c572fe5f4d6f81dd7a44c3340792, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733789693633 2024-12-10T00:14:59,135 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b33b5ada44b4850a302cb95d50117e1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733789693628 2024-12-10T00:14:59,135 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a220d4cefac349109a4aedcd9784229f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733789694805 2024-12-10T00:14:59,135 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba023bc01f74844b6df0a24c7955e7e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733789694805 2024-12-10T00:14:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,135 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d839c5735d403a9fd0f4947c5f1a29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733789696976 2024-12-10T00:14:59,135 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1c87c23d266419e9ce72327501a9dce, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733789696971 2024-12-10T00:14:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,153 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,154 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#177 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:59,154 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/6f9c191871a44401bbad0b50e9497372 is 50, key is test_row_0/B:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,165 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210824f114d3eef4f37b1ab51644172c52b_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,173 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210824f114d3eef4f37b1ab51644172c52b_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,173 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210824f114d3eef4f37b1ab51644172c52b_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742038_1214 (size=4469) 2024-12-10T00:14:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742037_1213 (size=12983) 2024-12-10T00:14:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:59,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:14:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121058b8594c95f2483b988c68de90f97e9a_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789698137/Put/seqid=0 2024-12-10T00:14:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742039_1215 (size=12454) 2024-12-10T00:14:59,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,353 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121058b8594c95f2483b988c68de90f97e9a_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121058b8594c95f2483b988c68de90f97e9a_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789699300, max timestamp=1733789699351Num uniques 859; 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789699300, max timestamp=1733789699351Num uniques 859; 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789699300, max timestamp=1733789699351Num uniques 859; 2024-12-10T00:14:59,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:14:59,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/899078051f5a43008cf168fd950b826b, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/899078051f5a43008cf168fd950b826b is 175, key is test_row_0/A:col10/1733789698137/Put/seqid=0 2024-12-10T00:14:59,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742040_1216 (size=31255) 2024-12-10T00:14:59,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/899078051f5a43008cf168fd950b826b 2024-12-10T00:14:59,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/97f96da54b064facb2729d830a6eef78 is 50, key is test_row_0/B:col10/1733789698137/Put/seqid=0 2024-12-10T00:14:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789759394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789759395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789759398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742041_1217 (size=12301) 2024-12-10T00:14:59,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/97f96da54b064facb2729d830a6eef78 2024-12-10T00:14:59,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/8d3b9cfc0a254057b037388ecc69ab12 is 50, key is test_row_0/C:col10/1733789698137/Put/seqid=0 2024-12-10T00:14:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742042_1218 (size=12301) 2024-12-10T00:14:59,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/8d3b9cfc0a254057b037388ecc69ab12 2024-12-10T00:14:59,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/899078051f5a43008cf168fd950b826b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b 2024-12-10T00:14:59,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b, entries=150, sequenceid=293, filesize=30.5 K 2024-12-10T00:14:59,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/97f96da54b064facb2729d830a6eef78 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78 2024-12-10T00:14:59,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T00:14:59,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/8d3b9cfc0a254057b037388ecc69ab12 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12 2024-12-10T00:14:59,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T00:14:59,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ce74af60a20cd73f95a0f2738491f1a3 in 176ms, sequenceid=293, compaction requested=true 2024-12-10T00:14:59,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:59,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T00:14:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:14:59,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-10T00:14:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:59,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:14:59,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:14:59,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121074d30ae3c6924c6da4b8c0bc1f940ee4_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789699512/Put/seqid=0 2024-12-10T00:14:59,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789759522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742043_1219 (size=14994) 2024-12-10T00:14:59,530 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:14:59,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789759529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789759529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,535 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121074d30ae3c6924c6da4b8c0bc1f940ee4_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074d30ae3c6924c6da4b8c0bc1f940ee4_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:14:59,536 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e5cba5ed66f241b1bd728f9454a544ba, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:14:59,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e5cba5ed66f241b1bd728f9454a544ba is 175, key is test_row_0/A:col10/1733789699512/Put/seqid=0 2024-12-10T00:14:59,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742044_1220 (size=39949) 2024-12-10T00:14:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-10T00:14:59,550 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:14:59,550 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:14:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-10T00:14:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:14:59,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:14:59,553 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:14:59,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:14:59,582 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:59,583 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/979fe1b88ad3466ab1c2d0efcd569f8e is 175, key is test_row_0/A:col10/1733789698121/Put/seqid=0 2024-12-10T00:14:59,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742045_1221 (size=31937) 2024-12-10T00:14:59,596 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/979fe1b88ad3466ab1c2d0efcd569f8e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e 2024-12-10T00:14:59,598 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/6f9c191871a44401bbad0b50e9497372 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6f9c191871a44401bbad0b50e9497372 2024-12-10T00:14:59,604 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into 6f9c191871a44401bbad0b50e9497372(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:59,604 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:59,604 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=12, startTime=1733789699125; duration=0sec 2024-12-10T00:14:59,604 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-10T00:14:59,604 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:59,604 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-10T00:14:59,605 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T00:14:59,608 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. because compaction request was cancelled 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:59,609 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into 979fe1b88ad3466ab1c2d0efcd569f8e(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:59,609 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=12, startTime=1733789699124; duration=0sec 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:14:59,609 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:59,610 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61601 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T00:14:59,610 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/C is initiating minor compaction (all files) 2024-12-10T00:14:59,610 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/C in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,611 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/2f8848044036446dba16b795a4f35b87, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=60.2 K 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. because compaction request was cancelled 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T00:14:59,611 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f8848044036446dba16b795a4f35b87, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789692970 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e21e04022a345a39a0a589907fecaf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733789693633 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. because compaction request was cancelled 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c42ad088fd45de9f27bb7c5686585b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733789694805 2024-12-10T00:14:59,612 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1d902806234d81b12d0d4f7cfc26de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733789696976 2024-12-10T00:14:59,613 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3b9cfc0a254057b037388ecc69ab12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733789698133 2024-12-10T00:14:59,636 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:14:59,639 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/f9032118dff64789bd4fdc76d599237c is 50, key is test_row_0/C:col10/1733789698137/Put/seqid=0 2024-12-10T00:14:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789759639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789759639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789759639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742046_1222 (size=13017) 2024-12-10T00:14:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:14:59,678 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/f9032118dff64789bd4fdc76d599237c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/f9032118dff64789bd4fdc76d599237c 2024-12-10T00:14:59,689 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/C of ce74af60a20cd73f95a0f2738491f1a3 into f9032118dff64789bd4fdc76d599237c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:14:59,689 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:14:59,689 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/C, priority=11, startTime=1733789699461; duration=0sec 2024-12-10T00:14:59,689 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:14:59,689 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:14:59,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789759850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789759851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:14:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789759851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:14:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:14:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:14:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:14:59,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:14:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:14:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:14:59,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:14:59,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e5cba5ed66f241b1bd728f9454a544ba 2024-12-10T00:14:59,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b591fb1666ad425e91b17b419637938b is 50, key is test_row_0/B:col10/1733789699512/Put/seqid=0 2024-12-10T00:14:59,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742047_1223 (size=12301) 2024-12-10T00:14:59,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b591fb1666ad425e91b17b419637938b 2024-12-10T00:14:59,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/e1115c1eccd2428e81f6bcc333e4041b is 50, key is test_row_0/C:col10/1733789699512/Put/seqid=0 2024-12-10T00:14:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742048_1224 (size=12301) 2024-12-10T00:15:00,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:15:00,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:15:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59432 deadline: 1733789760160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59406 deadline: 1733789760160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59424 deadline: 1733789760160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:00,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:15:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:15:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:15:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:00,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/e1115c1eccd2428e81f6bcc333e4041b 2024-12-10T00:15:00,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/e5cba5ed66f241b1bd728f9454a544ba as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba 2024-12-10T00:15:00,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba, entries=200, sequenceid=330, filesize=39.0 K 2024-12-10T00:15:00,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/b591fb1666ad425e91b17b419637938b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b 2024-12-10T00:15:00,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b, entries=150, sequenceid=330, filesize=12.0 K 2024-12-10T00:15:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/e1115c1eccd2428e81f6bcc333e4041b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/e1115c1eccd2428e81f6bcc333e4041b 2024-12-10T00:15:00,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/e1115c1eccd2428e81f6bcc333e4041b, entries=150, sequenceid=330, filesize=12.0 K 2024-12-10T00:15:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for ce74af60a20cd73f95a0f2738491f1a3 in 892ms, sequenceid=330, compaction requested=true 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:00,407 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:00,407 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce74af60a20cd73f95a0f2738491f1a3:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,408 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:00,408 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/B is initiating minor compaction (all files) 2024-12-10T00:15:00,408 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/B in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,408 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ce74af60a20cd73f95a0f2738491f1a3/A is initiating minor compaction (all files) 2024-12-10T00:15:00,409 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6f9c191871a44401bbad0b50e9497372, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=36.7 K 2024-12-10T00:15:00,409 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce74af60a20cd73f95a0f2738491f1a3/A in TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,409 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp, totalSize=100.7 K 2024-12-10T00:15:00,409 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba] 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9c191871a44401bbad0b50e9497372, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733789696976 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979fe1b88ad3466ab1c2d0efcd569f8e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733789696976 2024-12-10T00:15:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f96da54b064facb2729d830a6eef78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733789698133 2024-12-10T00:15:00,409 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899078051f5a43008cf168fd950b826b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733789698133 2024-12-10T00:15:00,410 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b591fb1666ad425e91b17b419637938b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733789699384 2024-12-10T00:15:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cba5ed66f241b1bd728f9454a544ba, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733789699384 2024-12-10T00:15:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,416 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,419 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,420 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/0209a4dae7b14839a8a307dc3e3c33de is 50, key is test_row_0/B:col10/1733789699512/Put/seqid=0 2024-12-10T00:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,421 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121077c49ae7709848cfaba7fbc67f8abc92_ce74af60a20cd73f95a0f2738491f1a3 store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,422 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121077c49ae7709848cfaba7fbc67f8abc92_ce74af60a20cd73f95a0f2738491f1a3, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,422 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121077c49ae7709848cfaba7fbc67f8abc92_ce74af60a20cd73f95a0f2738491f1a3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742049_1225 (size=13085) 2024-12-10T00:15:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742050_1226 (size=4469) 2024-12-10T00:15:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,432 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce74af60a20cd73f95a0f2738491f1a3#A#compaction#186 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,433 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/b04c6b74b2c24ca796e5de8cd6b72b07 is 175, key is test_row_0/A:col10/1733789699512/Put/seqid=0 2024-12-10T00:15:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,442 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/0209a4dae7b14839a8a307dc3e3c33de as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0209a4dae7b14839a8a307dc3e3c33de 2024-12-10T00:15:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,445 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-12-10T00:15:00,445 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,445 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,445 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,447 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-12-10T00:15:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,447 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,447 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,447 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,449 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/B of ce74af60a20cd73f95a0f2738491f1a3 into 0209a4dae7b14839a8a307dc3e3c33de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,449 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:15:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,449 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/B, priority=13, startTime=1733789700407; duration=0sec 2024-12-10T00:15:00,449 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:00,449 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:B 2024-12-10T00:15:00,449 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T00:15:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,450 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T00:15:00,450 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T00:15:00,450 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. because compaction request was cancelled 2024-12-10T00:15:00,450 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:C 2024-12-10T00:15:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,457 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-12-10T00:15:00,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742051_1227 (size=32039) 2024-12-10T00:15:00,458 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,458 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,458 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,470 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-12-10T00:15:00,471 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,471 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,471 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:00,484 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:00,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:15:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121010599dd7d1414b0689c96eea6fbaf5aa_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789699521/Put/seqid=0 2024-12-10T00:15:00,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742052_1228 (size=9914) 2024-12-10T00:15:00,671 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-12-10T00:15:00,672 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,672 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:00,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. as already flushing 2024-12-10T00:15:00,672 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-12-10T00:15:00,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,672 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,672 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-12-10T00:15:00,672 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,672 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:00,672 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:00,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:15:00,863 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/b04c6b74b2c24ca796e5de8cd6b72b07 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/b04c6b74b2c24ca796e5de8cd6b72b07 2024-12-10T00:15:00,872 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce74af60a20cd73f95a0f2738491f1a3/A of ce74af60a20cd73f95a0f2738491f1a3 into b04c6b74b2c24ca796e5de8cd6b72b07(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:00,873 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:15:00,873 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., storeName=ce74af60a20cd73f95a0f2738491f1a3/A, priority=13, startTime=1733789700407; duration=0sec 2024-12-10T00:15:00,873 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:00,873 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce74af60a20cd73f95a0f2738491f1a3:A 2024-12-10T00:15:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:00,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121010599dd7d1414b0689c96eea6fbaf5aa_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121010599dd7d1414b0689c96eea6fbaf5aa_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:00,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/5a3f0d6e4cf54c1d91eb9d69bd702477, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:00,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/5a3f0d6e4cf54c1d91eb9d69bd702477 is 175, key is test_row_0/A:col10/1733789699521/Put/seqid=0 2024-12-10T00:15:00,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742053_1229 (size=22561) 2024-12-10T00:15:01,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/5a3f0d6e4cf54c1d91eb9d69bd702477 2024-12-10T00:15:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/0c2504d449644d88bed8faf1704da2b4 is 50, key is test_row_0/B:col10/1733789699521/Put/seqid=0 2024-12-10T00:15:01,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742054_1230 (size=9857) 2024-12-10T00:15:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:15:01,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/0c2504d449644d88bed8faf1704da2b4 2024-12-10T00:15:01,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fd9325792c5c4f49a09b0b38583c5b5c is 50, key is test_row_0/C:col10/1733789699521/Put/seqid=0 2024-12-10T00:15:01,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742055_1231 (size=9857) 2024-12-10T00:15:02,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fd9325792c5c4f49a09b0b38583c5b5c 2024-12-10T00:15:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/5a3f0d6e4cf54c1d91eb9d69bd702477 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/5a3f0d6e4cf54c1d91eb9d69bd702477 2024-12-10T00:15:02,150 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/5a3f0d6e4cf54c1d91eb9d69bd702477, entries=100, sequenceid=342, filesize=22.0 K 2024-12-10T00:15:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/0c2504d449644d88bed8faf1704da2b4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0c2504d449644d88bed8faf1704da2b4 2024-12-10T00:15:02,155 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0c2504d449644d88bed8faf1704da2b4, entries=100, sequenceid=342, filesize=9.6 K 2024-12-10T00:15:02,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/fd9325792c5c4f49a09b0b38583c5b5c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fd9325792c5c4f49a09b0b38583c5b5c 2024-12-10T00:15:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fd9325792c5c4f49a09b0b38583c5b5c, entries=100, sequenceid=342, filesize=9.6 K 2024-12-10T00:15:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=20.13 KB/20610 for ce74af60a20cd73f95a0f2738491f1a3 in 1676ms, sequenceid=342, compaction requested=true 2024-12-10T00:15:02,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ce74af60a20cd73f95a0f2738491f1a3: 2024-12-10T00:15:02,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-10T00:15:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-10T00:15:02,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-10T00:15:02,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6090 sec 2024-12-10T00:15:02,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.6140 sec 2024-12-10T00:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T00:15:03,721 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:03,795 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:15:05,711 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-12-10T00:15:05,711 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-12-10T00:15:05,711 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:05,711 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:05,711 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,711 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,711 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,711 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:05,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3779 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3766 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1642 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4925 rows 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1637 2024-12-10T00:15:05,712 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4910 rows 2024-12-10T00:15:05,712 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:15:05,712 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:15:05,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:05,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:05,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:15:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:05,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789705715"}]},"ts":"1733789705715"} 2024-12-10T00:15:05,717 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:15:05,717 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:15:05,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:15:05,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, UNASSIGN}] 2024-12-10T00:15:05,720 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, UNASSIGN 2024-12-10T00:15:05,721 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:05,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, UNASSIGN because future has completed 2024-12-10T00:15:05,722 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:15:05,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:05,875 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing ce74af60a20cd73f95a0f2738491f1a3, disabling compactions & flushes 2024-12-10T00:15:05,875 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. after waiting 0 ms 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:05,875 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing ce74af60a20cd73f95a0f2738491f1a3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=A 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=B 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:05,875 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ce74af60a20cd73f95a0f2738491f1a3, store=C 2024-12-10T00:15:05,876 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:05,880 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210278272a2b4f940c38ccb6640b1f4c2f5_ce74af60a20cd73f95a0f2738491f1a3 is 50, key is test_row_0/A:col10/1733789705710/Put/seqid=0 2024-12-10T00:15:05,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742056_1232 (size=9914) 2024-12-10T00:15:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:06,285 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:06,294 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210278272a2b4f940c38ccb6640b1f4c2f5_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210278272a2b4f940c38ccb6640b1f4c2f5_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:06,296 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/ceb2ee66e5a348588dda63472bfd70d2, store: [table=TestAcidGuarantees family=A region=ce74af60a20cd73f95a0f2738491f1a3] 2024-12-10T00:15:06,297 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/ceb2ee66e5a348588dda63472bfd70d2 is 175, key is test_row_0/A:col10/1733789705710/Put/seqid=0 2024-12-10T00:15:06,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742057_1233 (size=22561) 2024-12-10T00:15:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:06,703 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/ceb2ee66e5a348588dda63472bfd70d2 2024-12-10T00:15:06,716 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/8f8403ec3d824123a9e6b727d3ea633e is 50, key is test_row_0/B:col10/1733789705710/Put/seqid=0 2024-12-10T00:15:06,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742058_1234 (size=9857) 2024-12-10T00:15:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:07,120 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/8f8403ec3d824123a9e6b727d3ea633e 2024-12-10T00:15:07,128 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/694caf978bf04963b031933c2ad2c1e9 is 50, key is test_row_0/C:col10/1733789705710/Put/seqid=0 2024-12-10T00:15:07,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742059_1235 (size=9857) 2024-12-10T00:15:07,534 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/694caf978bf04963b031933c2ad2c1e9 2024-12-10T00:15:07,543 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/A/ceb2ee66e5a348588dda63472bfd70d2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/ceb2ee66e5a348588dda63472bfd70d2 2024-12-10T00:15:07,547 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/ceb2ee66e5a348588dda63472bfd70d2, entries=100, sequenceid=351, filesize=22.0 K 2024-12-10T00:15:07,548 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/B/8f8403ec3d824123a9e6b727d3ea633e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/8f8403ec3d824123a9e6b727d3ea633e 2024-12-10T00:15:07,552 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/8f8403ec3d824123a9e6b727d3ea633e, entries=100, sequenceid=351, filesize=9.6 K 2024-12-10T00:15:07,553 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/.tmp/C/694caf978bf04963b031933c2ad2c1e9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/694caf978bf04963b031933c2ad2c1e9 2024-12-10T00:15:07,559 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/694caf978bf04963b031933c2ad2c1e9, entries=100, sequenceid=351, filesize=9.6 K 2024-12-10T00:15:07,561 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ce74af60a20cd73f95a0f2738491f1a3 in 1686ms, sequenceid=351, compaction requested=true 2024-12-10T00:15:07,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba] to archive 2024-12-10T00:15:07,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:07,579 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/d3ef58ed7ecf4b8bba946eb84d87ac95 2024-12-10T00:15:07,580 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/051102548fbb4d4b8e534817ea159e4f 2024-12-10T00:15:07,581 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1fe65c96096f4330b5dba54e55260fb2 2024-12-10T00:15:07,584 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/83a47f44110449b694720645ad4e2a61 2024-12-10T00:15:07,585 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e0587a9c998f4fa6bb6d92a825b7f0e4 2024-12-10T00:15:07,585 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/005b47d3a4f340afa5d7989e32ef741c 2024-12-10T00:15:07,590 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8187b0a979234006a981e636e38708a6 2024-12-10T00:15:07,590 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/35cb36c4d1d84f6e81755f26cf3d5a8b 2024-12-10T00:15:07,590 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/cc811250372148879e83a5b810e34f21 2024-12-10T00:15:07,591 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/da3844ce305a46c4a4265bed14c20679 2024-12-10T00:15:07,591 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/1500a7ee27264a109cb84da23f534215 2024-12-10T00:15:07,591 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0aff8d584c9b4c42ad0d915d75642d32 2024-12-10T00:15:07,591 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f496ed5e8c2c453e88467655e7b790cb 2024-12-10T00:15:07,591 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/f2cc1de5bfae4526a6d613f56d80713d 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/8b862238766e4e2983d052501315fad1 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/fba023bc01f74844b6df0a24c7955e7e 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/4db28f4782c94636ad04bca72a66513e 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/794ece6daf324b33b37a2053e719b040 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/0b33b5ada44b4850a302cb95d50117e1 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/899078051f5a43008cf168fd950b826b 2024-12-10T00:15:07,592 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/c1c87c23d266419e9ce72327501a9dce 2024-12-10T00:15:07,593 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/979fe1b88ad3466ab1c2d0efcd569f8e 2024-12-10T00:15:07,594 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/e5cba5ed66f241b1bd728f9454a544ba 2024-12-10T00:15:07,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e378439b137f4272be29d255d6a410d9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/071bedc24eec416f8a782c0a5cf0afa5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/084c91d7db104148a6ad96c9f88b490e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a86cad77cf5144fcb783201916d258e1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f5b7be038d54ed3a4520370e8cd677b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6f9c191871a44401bbad0b50e9497372, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b] to archive 2024-12-10T00:15:07,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:07,600 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e9794a6e7a6f4f689a2c6500c18766d2 2024-12-10T00:15:07,600 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/071bedc24eec416f8a782c0a5cf0afa5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/071bedc24eec416f8a782c0a5cf0afa5 2024-12-10T00:15:07,600 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/d5d64d6525f649e6ae90ddfb5a587777 2024-12-10T00:15:07,600 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e378439b137f4272be29d255d6a410d9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/e378439b137f4272be29d255d6a410d9 2024-12-10T00:15:07,600 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a56c9dbac69a4c7b88be89814552adb0 2024-12-10T00:15:07,601 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/c55cf36884b24bbb8cd211c7c95b2bbe 2024-12-10T00:15:07,601 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/1835025b9a084e36afdc10b2524146b4 2024-12-10T00:15:07,601 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/ed08665c9597417695854f0c01ff44d1 2024-12-10T00:15:07,601 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/084c91d7db104148a6ad96c9f88b490e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/084c91d7db104148a6ad96c9f88b490e 2024-12-10T00:15:07,601 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6382c859135d459d88d956c23525e828 2024-12-10T00:15:07,602 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/eeffa6151e1048c0839e9277ebeae3ab 2024-12-10T00:15:07,602 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b466997cc22c4818a5242add1c741b97 2024-12-10T00:15:07,603 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f5b7be038d54ed3a4520370e8cd677b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f5b7be038d54ed3a4520370e8cd677b 2024-12-10T00:15:07,603 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/fcdb8a1884f6435d9f3b869a1e1b7e7f 2024-12-10T00:15:07,603 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b2d1f79edb95481388d5a2d26b09b3a5 2024-12-10T00:15:07,603 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/3f73c572fe5f4d6f81dd7a44c3340792 2024-12-10T00:15:07,603 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/59ca48ac1d94468e80191e11780fe986 2024-12-10T00:15:07,604 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6f9c191871a44401bbad0b50e9497372 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/6f9c191871a44401bbad0b50e9497372 2024-12-10T00:15:07,604 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a220d4cefac349109a4aedcd9784229f 2024-12-10T00:15:07,604 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b0d839c5735d403a9fd0f4947c5f1a29 2024-12-10T00:15:07,604 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/97f96da54b064facb2729d830a6eef78 2024-12-10T00:15:07,604 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/b591fb1666ad425e91b17b419637938b 2024-12-10T00:15:07,605 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a86cad77cf5144fcb783201916d258e1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/a86cad77cf5144fcb783201916d258e1 2024-12-10T00:15:07,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/23d6adf5e40f472bad66405b0d732c88, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0e71528178ca4a77be645c57c22743e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/cf62bf9ef1274e14812c74a7102ef2ee, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/35a125c14bc244e28fcdee72fdd1b5bc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/2f8848044036446dba16b795a4f35b87, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12] to archive 2024-12-10T00:15:07,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0bc94dd03c254d22a9277ce9df3ea115 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3eb1c8ed713742d39485f5465d1f061f 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fc4e5bc7f642478c99666aaeb6d122fa 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d18c554d5c534d03a06a63cef855d1e7 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0e71528178ca4a77be645c57c22743e5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/0e71528178ca4a77be645c57c22743e5 2024-12-10T00:15:07,615 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/23d6adf5e40f472bad66405b0d732c88 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/23d6adf5e40f472bad66405b0d732c88 2024-12-10T00:15:07,616 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/65d00a72343248fcb2c803ce73234d63 2024-12-10T00:15:07,617 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/9b455942b31b4d43b25f1a390ee89ce8 2024-12-10T00:15:07,617 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/d3245bbd84514dd39d18e36e16e12f04 2024-12-10T00:15:07,617 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/35a125c14bc244e28fcdee72fdd1b5bc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/35a125c14bc244e28fcdee72fdd1b5bc 2024-12-10T00:15:07,617 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/ef2a6287386b4f07864e24220c51102d 2024-12-10T00:15:07,617 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/cf62bf9ef1274e14812c74a7102ef2ee to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/cf62bf9ef1274e14812c74a7102ef2ee 2024-12-10T00:15:07,618 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/7ed104fd865548c2b0dba190e82f12bb 2024-12-10T00:15:07,620 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/4cb3a5478ec9452d936439b0537b1d86 2024-12-10T00:15:07,618 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/77f1a85fd3d440b886b4ab07aeb10d82 2024-12-10T00:15:07,621 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fdfaa53c03424476b802b7cc4c07c81f 2024-12-10T00:15:07,626 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/2f8848044036446dba16b795a4f35b87 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/2f8848044036446dba16b795a4f35b87 2024-12-10T00:15:07,626 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/19c42ad088fd45de9f27bb7c5686585b 2024-12-10T00:15:07,626 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/3e21e04022a345a39a0a589907fecaf1 2024-12-10T00:15:07,626 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/dc1d902806234d81b12d0d4f7cfc26de 2024-12-10T00:15:07,627 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/8d3b9cfc0a254057b037388ecc69ab12 2024-12-10T00:15:07,640 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-12-10T00:15:07,641 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. 2024-12-10T00:15:07,641 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for ce74af60a20cd73f95a0f2738491f1a3: Waiting for close lock at 1733789705875Running coprocessor pre-close hooks at 1733789705875Disabling compacts and flushes for region at 1733789705875Disabling writes for close at 1733789705875Obtaining lock to block concurrent updates at 1733789705875Preparing flush snapshotting stores in ce74af60a20cd73f95a0f2738491f1a3 at 1733789705875Finished memstore snapshotting TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733789705876 (+1 ms)Flushing stores of TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3. at 1733789705876Flushing ce74af60a20cd73f95a0f2738491f1a3/A: creating writer at 1733789705876Flushing mob file ce74af60a20cd73f95a0f2738491f1a3/A: appending metadata at 1733789705880 (+4 ms)Flushing mob file ce74af60a20cd73f95a0f2738491f1a3/A: closing flushed file at 1733789705880Flushing ce74af60a20cd73f95a0f2738491f1a3/A: appending metadata at 1733789706296 (+416 ms)Flushing ce74af60a20cd73f95a0f2738491f1a3/A: closing flushed file at 1733789706296Flushing ce74af60a20cd73f95a0f2738491f1a3/B: creating writer at 1733789706712 (+416 ms)Flushing ce74af60a20cd73f95a0f2738491f1a3/B: appending metadata at 1733789706716 (+4 ms)Flushing ce74af60a20cd73f95a0f2738491f1a3/B: closing flushed file at 1733789706716Flushing ce74af60a20cd73f95a0f2738491f1a3/C: creating writer at 1733789707124 (+408 ms)Flushing ce74af60a20cd73f95a0f2738491f1a3/C: appending metadata at 1733789707128 (+4 ms)Flushing ce74af60a20cd73f95a0f2738491f1a3/C: closing flushed file at 1733789707128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17efbd44: reopening flushed file at 1733789707542 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43bbeee5: reopening flushed file at 1733789707547 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cf4aca0: reopening flushed file at 1733789707552 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ce74af60a20cd73f95a0f2738491f1a3 in 1686ms, sequenceid=351, compaction requested=true at 1733789707561 (+9 ms)Writing region close event to WAL at 1733789707637 (+76 ms)Running coprocessor post-close hooks at 1733789707641 (+4 ms)Closed at 1733789707641 2024-12-10T00:15:07,644 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,644 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ce74af60a20cd73f95a0f2738491f1a3, regionState=CLOSED 2024-12-10T00:15:07,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:07,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-10T00:15:07,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure ce74af60a20cd73f95a0f2738491f1a3, server=a67c4886b4f7,36911,1733789645811 in 1.9240 sec 2024-12-10T00:15:07,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-10T00:15:07,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ce74af60a20cd73f95a0f2738491f1a3, UNASSIGN in 1.9280 sec 2024-12-10T00:15:07,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-10T00:15:07,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9320 sec 2024-12-10T00:15:07,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789707654"}]},"ts":"1733789707654"} 2024-12-10T00:15:07,656 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:15:07,656 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:15:07,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9440 sec 2024-12-10T00:15:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-10T00:15:07,860 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:07,861 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:15:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,863 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T00:15:07,863 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,866 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,869 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits] 2024-12-10T00:15:07,873 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/ceb2ee66e5a348588dda63472bfd70d2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/ceb2ee66e5a348588dda63472bfd70d2 2024-12-10T00:15:07,873 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/b04c6b74b2c24ca796e5de8cd6b72b07 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/b04c6b74b2c24ca796e5de8cd6b72b07 2024-12-10T00:15:07,873 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/5a3f0d6e4cf54c1d91eb9d69bd702477 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/A/5a3f0d6e4cf54c1d91eb9d69bd702477 2024-12-10T00:15:07,877 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/8f8403ec3d824123a9e6b727d3ea633e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/8f8403ec3d824123a9e6b727d3ea633e 2024-12-10T00:15:07,877 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0209a4dae7b14839a8a307dc3e3c33de to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0209a4dae7b14839a8a307dc3e3c33de 2024-12-10T00:15:07,877 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0c2504d449644d88bed8faf1704da2b4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/B/0c2504d449644d88bed8faf1704da2b4 2024-12-10T00:15:07,881 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/e1115c1eccd2428e81f6bcc333e4041b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/e1115c1eccd2428e81f6bcc333e4041b 2024-12-10T00:15:07,881 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/694caf978bf04963b031933c2ad2c1e9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/694caf978bf04963b031933c2ad2c1e9 2024-12-10T00:15:07,881 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fd9325792c5c4f49a09b0b38583c5b5c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/fd9325792c5c4f49a09b0b38583c5b5c 2024-12-10T00:15:07,881 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/f9032118dff64789bd4fdc76d599237c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/C/f9032118dff64789bd4fdc76d599237c 2024-12-10T00:15:07,884 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits/354.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3/recovered.edits/354.seqid 2024-12-10T00:15:07,885 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,885 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:15:07,886 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:15:07,886 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T00:15:07,894 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210278272a2b4f940c38ccb6640b1f4c2f5_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210278272a2b4f940c38ccb6640b1f4c2f5_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,894 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121010599dd7d1414b0689c96eea6fbaf5aa_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121010599dd7d1414b0689c96eea6fbaf5aa_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102fa54ebf2769425ca093e7447d0f0106_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102fa54ebf2769425ca093e7447d0f0106_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121046716fdc11c945af8379a1a95c426ac2_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121046716fdc11c945af8379a1a95c426ac2_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104faa9e5bda92411e8ae0b52ddfaeac85_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104faa9e5bda92411e8ae0b52ddfaeac85_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072df177e2c254df7bd63a6bc1d33db8d_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072df177e2c254df7bd63a6bc1d33db8d_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064b905a69541424ca85984121f7209bb_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064b905a69541424ca85984121f7209bb_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,895 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121058b8594c95f2483b988c68de90f97e9a_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121058b8594c95f2483b988c68de90f97e9a_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,896 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210926af22a34514b5da885ed9c4a8c310c_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210926af22a34514b5da885ed9c4a8c310c_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,896 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109378d66c12a341d59b6c444b9c3af18f_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109378d66c12a341d59b6c444b9c3af18f_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,896 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210955761175dc24b18a2b863360a6bf571_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210955761175dc24b18a2b863360a6bf571_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,897 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108a9c6290168e460f8dcc001aac63c8a2_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108a9c6290168e460f8dcc001aac63c8a2_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,897 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074d30ae3c6924c6da4b8c0bc1f940ee4_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074d30ae3c6924c6da4b8c0bc1f940ee4_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,897 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121098b3f611f2bc4b36879750e39c168c19_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121098b3f611f2bc4b36879750e39c168c19_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,897 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b4c6fb57374e42e1945be7cd87eb7154_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b4c6fb57374e42e1945be7cd87eb7154_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,897 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccfe5aeeab5848a587f1f48d21e64bc1_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccfe5aeeab5848a587f1f48d21e64bc1_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,898 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f30d3838d5704c7a99738c2b15896a04_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f30d3838d5704c7a99738c2b15896a04_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,898 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db94dee420d4451c87e8a82988254c00_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210db94dee420d4451c87e8a82988254c00_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,899 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27ef003ef0e494b9b37da22f719317d_ce74af60a20cd73f95a0f2738491f1a3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27ef003ef0e494b9b37da22f719317d_ce74af60a20cd73f95a0f2738491f1a3 2024-12-10T00:15:07,899 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:15:07,902 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,905 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:15:07,907 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:15:07,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:15:07,910 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789707910"}]},"ts":"9223372036854775807"} 2024-12-10T00:15:07,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:15:07,913 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ce74af60a20cd73f95a0f2738491f1a3, NAME => 'TestAcidGuarantees,,1733789677677.ce74af60a20cd73f95a0f2738491f1a3.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:15:07,913 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:15:07,913 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789707913"}]},"ts":"9223372036854775807"} 2024-12-10T00:15:07,915 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:15:07,916 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:07,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-10T00:15:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T00:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:15:07,971 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:07,983 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255 (was 256), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=475 (was 510), ProcessCount=11 (was 11), AvailableMemoryMB=2907 (was 3403) 2024-12-10T00:15:07,997 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=255, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=475, ProcessCount=11, AvailableMemoryMB=2908 2024-12-10T00:15:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:15:08,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:15:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:08,005 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:15:08,005 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:08,006 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:15:08,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-10T00:15:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:08,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742060_1236 (size=1061) 2024-12-10T00:15:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:08,416 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:15:08,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742061_1237 (size=53) 2024-12-10T00:15:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing de1b03cb10a30c0ceaa7548777a2855e, disabling compactions & flushes 2024-12-10T00:15:08,823 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. after waiting 0 ms 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:08,823 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:08,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for de1b03cb10a30c0ceaa7548777a2855e: Waiting for close lock at 1733789708823Disabling compacts and flushes for region at 1733789708823Disabling writes for close at 1733789708823Writing region close event to WAL at 1733789708823Closed at 1733789708823 2024-12-10T00:15:08,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:15:08,825 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789708824"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789708824"}]},"ts":"1733789708824"} 2024-12-10T00:15:08,826 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:15:08,827 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:15:08,827 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789708827"}]},"ts":"1733789708827"} 2024-12-10T00:15:08,829 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:15:08,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, ASSIGN}] 2024-12-10T00:15:08,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, ASSIGN 2024-12-10T00:15:08,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:15:08,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=de1b03cb10a30c0ceaa7548777a2855e, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:08,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, ASSIGN because future has completed 2024-12-10T00:15:08,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:09,138 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:09,138 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:15:09,139 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,139 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:15:09,139 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,139 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,140 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,141 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:09,141 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de1b03cb10a30c0ceaa7548777a2855e columnFamilyName A 2024-12-10T00:15:09,142 DEBUG [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:09,142 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(327): Store=de1b03cb10a30c0ceaa7548777a2855e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:09,142 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,143 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:09,143 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de1b03cb10a30c0ceaa7548777a2855e columnFamilyName B 2024-12-10T00:15:09,143 DEBUG [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:09,144 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(327): Store=de1b03cb10a30c0ceaa7548777a2855e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:09,144 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,145 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:09,145 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de1b03cb10a30c0ceaa7548777a2855e columnFamilyName C 2024-12-10T00:15:09,145 DEBUG [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:09,145 INFO [StoreOpener-de1b03cb10a30c0ceaa7548777a2855e-1 {}] regionserver.HStore(327): Store=de1b03cb10a30c0ceaa7548777a2855e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:09,145 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:09,145 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,146 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,146 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,147 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,147 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,148 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:15:09,149 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:09,151 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:15:09,152 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened de1b03cb10a30c0ceaa7548777a2855e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65712517, jitterRate=-0.020807191729545593}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:15:09,152 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:09,153 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for de1b03cb10a30c0ceaa7548777a2855e: Running coprocessor pre-open hook at 1733789709139Writing region info on filesystem at 1733789709139Initializing all the Stores at 1733789709140 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789709140Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789709140Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789709140Cleaning up temporary data from old regions at 1733789709147 (+7 ms)Running coprocessor post-open hooks at 1733789709152 (+5 ms)Region opened successfully at 1733789709153 (+1 ms) 2024-12-10T00:15:09,154 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., pid=63, masterSystemTime=1733789709135 2024-12-10T00:15:09,157 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:09,157 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:09,158 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=de1b03cb10a30c0ceaa7548777a2855e, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:09,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:09,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-10T00:15:09,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 in 178 msec 2024-12-10T00:15:09,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-10T00:15:09,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, ASSIGN in 334 msec 2024-12-10T00:15:09,168 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:15:09,168 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789709168"}]},"ts":"1733789709168"} 2024-12-10T00:15:09,170 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:15:09,171 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:15:09,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-12-10T00:15:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-10T00:15:10,160 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:10,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba3b8cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d04bfdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,164 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,167 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:15:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:15:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2663b8f6 2024-12-10T00:15:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:15:10,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:15:10,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,171 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3f7a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f2ad93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b2cdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,176 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a212926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,177 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a63dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc5afa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe63be2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d11d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,184 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1e74d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,186 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e2d54c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c67455b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b96ae40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40476d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,193 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ba669b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed6dd65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,196 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b44b495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,198 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,201 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39396490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,203 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8c79e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:10,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:10,206 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:10,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4fc787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:10,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683ee30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:10,210 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,211 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,211 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,211 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,212 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,213 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-10T00:15:10,231 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:10,232 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:10,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:10,239 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:10,240 DEBUG [Thread-1112 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,240 DEBUG [Thread-1112 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:10,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:10,255 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,256 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,271 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:10,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:15:10,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/51faebf1352c47d2807843a13dfdfb98 is 50, key is test_row_0/A:col10/1733789710240/Put/seqid=0 2024-12-10T00:15:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789770298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789770299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789770300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789710289, max timestamp=1733789710304Num uniques 859; 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789710289, max timestamp=1733789710304Num uniques 859; 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789710289, max timestamp=1733789710304Num uniques 859; 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:10,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742062_1238 (size=12001) 2024-12-10T00:15:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:10,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789770354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789770358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:10,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789770420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789770420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789770420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789770469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789770469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789770629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789770630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789770630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789770679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789770679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:10,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:10,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/51faebf1352c47d2807843a13dfdfb98 2024-12-10T00:15:10,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/6fc7d714efad48c79852cd2895a94527 is 50, key is test_row_0/B:col10/1733789710240/Put/seqid=0 2024-12-10T00:15:10,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742063_1239 (size=12001) 2024-12-10T00:15:10,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:10,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789770941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789770941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789770941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789770989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:10,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789770992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:10,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:11,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/6fc7d714efad48c79852cd2895a94527 2024-12-10T00:15:11,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:11,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:11,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:11,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:11,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:11,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/68724f6e75954a31baf086182d8088ec is 50, key is test_row_0/C:col10/1733789710240/Put/seqid=0 2024-12-10T00:15:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742064_1240 (size=12001) 2024-12-10T00:15:11,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/68724f6e75954a31baf086182d8088ec 2024-12-10T00:15:11,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/51faebf1352c47d2807843a13dfdfb98 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98 2024-12-10T00:15:11,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T00:15:11,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/6fc7d714efad48c79852cd2895a94527 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527 2024-12-10T00:15:11,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T00:15:11,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/68724f6e75954a31baf086182d8088ec as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec 2024-12-10T00:15:11,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec, entries=150, sequenceid=13, filesize=11.7 K 2024-12-10T00:15:11,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for de1b03cb10a30c0ceaa7548777a2855e in 982ms, sequenceid=13, compaction requested=false 2024-12-10T00:15:11,222 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T00:15:11,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:11,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T00:15:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:11,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:11,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/be94343e6601422194c889080c90a087 is 50, key is test_row_0/A:col10/1733789710350/Put/seqid=0 2024-12-10T00:15:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742065_1241 (size=12001) 2024-12-10T00:15:11,336 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/be94343e6601422194c889080c90a087 2024-12-10T00:15:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/ef629794d72c418aa7af254bfdbac2f0 is 50, key is test_row_0/B:col10/1733789710350/Put/seqid=0 2024-12-10T00:15:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:11,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742066_1242 (size=12001) 2024-12-10T00:15:11,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789771473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789771476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789771477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789771499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789771505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789771593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789771593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789771593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,785 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/ef629794d72c418aa7af254bfdbac2f0 2024-12-10T00:15:11,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/181e8d783f9649a0ad6fc9f20c60ed4c is 50, key is test_row_0/C:col10/1733789710350/Put/seqid=0 2024-12-10T00:15:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789771800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789771809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789771809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:11,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742067_1243 (size=12001) 2024-12-10T00:15:12,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789772110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789772119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789772119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,263 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/181e8d783f9649a0ad6fc9f20c60ed4c 2024-12-10T00:15:12,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/be94343e6601422194c889080c90a087 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087 2024-12-10T00:15:12,272 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:15:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/ef629794d72c418aa7af254bfdbac2f0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0 2024-12-10T00:15:12,280 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:15:12,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/181e8d783f9649a0ad6fc9f20c60ed4c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c 2024-12-10T00:15:12,286 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:15:12,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=67.09 KB/68700 for de1b03cb10a30c0ceaa7548777a2855e in 979ms, sequenceid=47, compaction requested=false 2024-12-10T00:15:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-10T00:15:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-10T00:15:12,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-10T00:15:12,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-12-10T00:15:12,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0670 sec 2024-12-10T00:15:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T00:15:12,380 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:12,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-10T00:15:12,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T00:15:12,384 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:12,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T00:15:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:12,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/f527898a7a2d460ba63ebc44c7d55d1f is 50, key is test_row_0/A:col10/1733789712519/Put/seqid=0 2024-12-10T00:15:12,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T00:15:12,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:12,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742068_1244 (size=12001) 2024-12-10T00:15:12,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/f527898a7a2d460ba63ebc44c7d55d1f 2024-12-10T00:15:12,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/55b3f6b75ce4417d8cdde1bbb2e19595 is 50, key is test_row_0/B:col10/1733789712519/Put/seqid=0 2024-12-10T00:15:12,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742069_1245 (size=12001) 2024-12-10T00:15:12,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/55b3f6b75ce4417d8cdde1bbb2e19595 2024-12-10T00:15:12,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789772562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789772562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6ee0856637ce4f5a97a10a4f5f443912 is 50, key is test_row_0/C:col10/1733789712519/Put/seqid=0 2024-12-10T00:15:12,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742070_1246 (size=12001) 2024-12-10T00:15:12,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789772620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789772629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789772629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789772670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789772670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T00:15:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:12,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T00:15:12,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T00:15:12,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:12,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:12,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789772880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789772880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:12,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6ee0856637ce4f5a97a10a4f5f443912 2024-12-10T00:15:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/f527898a7a2d460ba63ebc44c7d55d1f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f 2024-12-10T00:15:12,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T00:15:12,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/55b3f6b75ce4417d8cdde1bbb2e19595 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595 2024-12-10T00:15:12,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T00:15:12,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6ee0856637ce4f5a97a10a4f5f443912 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912 2024-12-10T00:15:12,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T00:15:12,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for de1b03cb10a30c0ceaa7548777a2855e in 473ms, sequenceid=62, compaction requested=true 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:12,995 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:12,995 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:12,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T00:15:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,996 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:12,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T00:15:12,996 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:12,996 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:12,997 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.2 K 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:12,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:12,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:12,997 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:12,997 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.2 K 2024-12-10T00:15:12,997 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51faebf1352c47d2807843a13dfdfb98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733789710229 2024-12-10T00:15:12,998 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc7d714efad48c79852cd2895a94527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733789710229 2024-12-10T00:15:12,998 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting be94343e6601422194c889080c90a087, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789710322 2024-12-10T00:15:12,998 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ef629794d72c418aa7af254bfdbac2f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789710322 2024-12-10T00:15:12,999 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f527898a7a2d460ba63ebc44c7d55d1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789711468 2024-12-10T00:15:12,999 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b3f6b75ce4417d8cdde1bbb2e19595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789711468 2024-12-10T00:15:13,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/ee955a48c1e54ac397f325cf58b14e33 is 50, key is test_row_0/A:col10/1733789712554/Put/seqid=0 2024-12-10T00:15:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T00:15:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742071_1247 (size=12001) 2024-12-10T00:15:13,035 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/ee955a48c1e54ac397f325cf58b14e33 2024-12-10T00:15:13,041 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:13,042 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/c751d62ddd774b3b982881a44d4727c2 is 50, key is test_row_0/A:col10/1733789712519/Put/seqid=0 2024-12-10T00:15:13,053 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#205 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:13,053 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4c135be216a84bcbbd61b09b7e4f5345 is 50, key is test_row_0/B:col10/1733789712519/Put/seqid=0 2024-12-10T00:15:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/edda83b93a76411fa08eadf672d74b71 is 50, key is test_row_0/B:col10/1733789712554/Put/seqid=0 2024-12-10T00:15:13,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742072_1248 (size=12104) 2024-12-10T00:15:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742073_1249 (size=12104) 2024-12-10T00:15:13,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742074_1250 (size=12001) 2024-12-10T00:15:13,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/edda83b93a76411fa08eadf672d74b71 2024-12-10T00:15:13,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/92c1bf2a8a764d83831d44163c32f1ac is 50, key is test_row_0/C:col10/1733789712554/Put/seqid=0 2024-12-10T00:15:13,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742075_1251 (size=12001) 2024-12-10T00:15:13,183 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/92c1bf2a8a764d83831d44163c32f1ac 2024-12-10T00:15:13,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/ee955a48c1e54ac397f325cf58b14e33 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33 2024-12-10T00:15:13,192 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T00:15:13,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:13,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/edda83b93a76411fa08eadf672d74b71 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71 2024-12-10T00:15:13,195 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:15:13,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T00:15:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/92c1bf2a8a764d83831d44163c32f1ac as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac 2024-12-10T00:15:13,213 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac, entries=150, sequenceid=79, filesize=11.7 K 2024-12-10T00:15:13,214 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=53.67 KB/54960 for de1b03cb10a30c0ceaa7548777a2855e in 218ms, sequenceid=79, compaction requested=true 2024-12-10T00:15:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-10T00:15:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-10T00:15:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:13,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T00:15:13,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-10T00:15:13,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 832 msec 2024-12-10T00:15:13,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:13,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:13,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 840 msec 2024-12-10T00:15:13,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/8fc12432c6fe43e3b843c27ed8e78e1c is 50, key is test_row_0/A:col10/1733789713211/Put/seqid=0 2024-12-10T00:15:13,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742076_1252 (size=12001) 2024-12-10T00:15:13,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789773296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789773300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789773410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789773410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,502 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/c751d62ddd774b3b982881a44d4727c2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/c751d62ddd774b3b982881a44d4727c2 2024-12-10T00:15:13,515 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into c751d62ddd774b3b982881a44d4727c2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:13,515 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:13,515 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789712995; duration=0sec 2024-12-10T00:15:13,515 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:13,516 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:13,516 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:13,517 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:13,517 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:13,517 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,518 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=46.9 K 2024-12-10T00:15:13,518 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68724f6e75954a31baf086182d8088ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733789710229 2024-12-10T00:15:13,518 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181e8d783f9649a0ad6fc9f20c60ed4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789710322 2024-12-10T00:15:13,519 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee0856637ce4f5a97a10a4f5f443912, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789711468 2024-12-10T00:15:13,519 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c1bf2a8a764d83831d44163c32f1ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733789712547 2024-12-10T00:15:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T00:15:13,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:13,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-10T00:15:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:13,525 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:13,526 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:13,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:13,539 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4c135be216a84bcbbd61b09b7e4f5345 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4c135be216a84bcbbd61b09b7e4f5345 2024-12-10T00:15:13,539 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:13,540 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/828b795fed374a0bafa4025216769ed9 is 50, key is test_row_0/C:col10/1733789712554/Put/seqid=0 2024-12-10T00:15:13,547 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 4c135be216a84bcbbd61b09b7e4f5345(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:13,547 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:13,547 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789712995; duration=0sec 2024-12-10T00:15:13,547 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:13,547 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:13,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742077_1253 (size=12139) 2024-12-10T00:15:13,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789773619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789773630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,632 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789773630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789773639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789773639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,641 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,641 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/8fc12432c6fe43e3b843c27ed8e78e1c 2024-12-10T00:15:13,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/36bc9d9142bc474d865eec9da2bc2809 is 50, key is test_row_0/B:col10/1733789713211/Put/seqid=0 2024-12-10T00:15:13,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742078_1254 (size=12001) 2024-12-10T00:15:13,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T00:15:13,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:13,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T00:15:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:13,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789773929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789773939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:13,979 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/828b795fed374a0bafa4025216769ed9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/828b795fed374a0bafa4025216769ed9 2024-12-10T00:15:13,982 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 828b795fed374a0bafa4025216769ed9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:13,983 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:13,983 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=12, startTime=1733789712995; duration=0sec 2024-12-10T00:15:13,983 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:13,983 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:13,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T00:15:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:13,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:14,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/36bc9d9142bc474d865eec9da2bc2809 2024-12-10T00:15:14,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f15dbb807834873b5f25d38ce010527 is 50, key is test_row_0/C:col10/1733789713211/Put/seqid=0 2024-12-10T00:15:14,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742079_1255 (size=12001) 2024-12-10T00:15:14,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f15dbb807834873b5f25d38ce010527 2024-12-10T00:15:14,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/8fc12432c6fe43e3b843c27ed8e78e1c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c 2024-12-10T00:15:14,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c, entries=150, sequenceid=92, filesize=11.7 K 2024-12-10T00:15:14,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/36bc9d9142bc474d865eec9da2bc2809 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809 2024-12-10T00:15:14,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809, entries=150, sequenceid=92, filesize=11.7 K 2024-12-10T00:15:14,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f15dbb807834873b5f25d38ce010527 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527 2024-12-10T00:15:14,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527, entries=150, sequenceid=92, filesize=11.7 K 2024-12-10T00:15:14,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for de1b03cb10a30c0ceaa7548777a2855e in 893ms, sequenceid=92, compaction requested=true 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:14,111 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:14,111 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:14,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:14,113 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:14,113 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:14,113 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:14,113 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4c135be216a84bcbbd61b09b7e4f5345, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.3 K 2024-12-10T00:15:14,114 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:14,114 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:14,114 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:14,114 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/c751d62ddd774b3b982881a44d4727c2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.3 K 2024-12-10T00:15:14,114 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c135be216a84bcbbd61b09b7e4f5345, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789711468 2024-12-10T00:15:14,114 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c751d62ddd774b3b982881a44d4727c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733789711468 2024-12-10T00:15:14,115 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee955a48c1e54ac397f325cf58b14e33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733789712547 2024-12-10T00:15:14,115 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting edda83b93a76411fa08eadf672d74b71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733789712547 2024-12-10T00:15:14,116 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc12432c6fe43e3b843c27ed8e78e1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733789713204 2024-12-10T00:15:14,116 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bc9d9142bc474d865eec9da2bc2809, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733789713204 2024-12-10T00:15:14,126 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:14,127 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:14,127 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/719570579eab457a85c5cccd86d72c00 is 50, key is test_row_0/A:col10/1733789713211/Put/seqid=0 2024-12-10T00:15:14,127 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/86c72046315142ed85bc4ad00bc08dd1 is 50, key is test_row_0/B:col10/1733789713211/Put/seqid=0 2024-12-10T00:15:14,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742080_1256 (size=12207) 2024-12-10T00:15:14,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T00:15:14,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:14,138 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:14,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742081_1257 (size=12207) 2024-12-10T00:15:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:14,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5a4df617146b48d7863ee9aa305e6686 is 50, key is test_row_0/A:col10/1733789713286/Put/seqid=0 2024-12-10T00:15:14,159 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/86c72046315142ed85bc4ad00bc08dd1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/86c72046315142ed85bc4ad00bc08dd1 2024-12-10T00:15:14,165 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 86c72046315142ed85bc4ad00bc08dd1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:14,166 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:14,166 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789714111; duration=0sec 2024-12-10T00:15:14,166 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:14,166 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:14,166 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T00:15:14,167 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T00:15:14,167 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T00:15:14,167 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. because compaction request was cancelled 2024-12-10T00:15:14,167 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:14,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742082_1258 (size=12001) 2024-12-10T00:15:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:14,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789774465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:14,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789774466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:14,546 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/719570579eab457a85c5cccd86d72c00 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/719570579eab457a85c5cccd86d72c00 2024-12-10T00:15:14,551 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 719570579eab457a85c5cccd86d72c00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:14,551 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:14,551 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789714111; duration=0sec 2024-12-10T00:15:14,551 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:14,551 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:14,569 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5a4df617146b48d7863ee9aa305e6686 2024-12-10T00:15:14,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/5b7c4bd1f2e142d0ab54a68e7c1706ed is 50, key is test_row_0/B:col10/1733789713286/Put/seqid=0 2024-12-10T00:15:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789774579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789774580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742083_1259 (size=12001) 2024-12-10T00:15:14,592 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/5b7c4bd1f2e142d0ab54a68e7c1706ed 2024-12-10T00:15:14,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 is 50, key is test_row_0/C:col10/1733789713286/Put/seqid=0 2024-12-10T00:15:14,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742084_1260 (size=12001) 2024-12-10T00:15:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789774790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789774790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,012 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 2024-12-10T00:15:15,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5a4df617146b48d7863ee9aa305e6686 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686 2024-12-10T00:15:15,020 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T00:15:15,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/5b7c4bd1f2e142d0ab54a68e7c1706ed as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed 2024-12-10T00:15:15,025 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T00:15:15,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 2024-12-10T00:15:15,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5, entries=150, sequenceid=114, filesize=11.7 K 2024-12-10T00:15:15,031 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for de1b03cb10a30c0ceaa7548777a2855e in 893ms, sequenceid=114, compaction requested=true 2024-12-10T00:15:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:15,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-10T00:15:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-10T00:15:15,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T00:15:15,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-12-10T00:15:15,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.5120 sec 2024-12-10T00:15:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:15,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:15,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:15,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6c8f9d15c52b429184cf501605d83c99 is 50, key is test_row_0/A:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:15,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742085_1261 (size=16781) 2024-12-10T00:15:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789775128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789775129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789775240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789775240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789775450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789775450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6c8f9d15c52b429184cf501605d83c99 2024-12-10T00:15:15,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4f871e73fcb74991b9cd67c0fe3ca08e is 50, key is test_row_0/B:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:15,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742086_1262 (size=12051) 2024-12-10T00:15:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789775650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,654 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789775660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789775660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,663 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,663 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T00:15:15,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:15,670 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-10T00:15:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:15,672 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:15,673 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:15,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:15,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789775761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789775761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:15,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T00:15:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:15,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4f871e73fcb74991b9cd67c0fe3ca08e 2024-12-10T00:15:15,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T00:15:15,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:15,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:15,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3e8610ca807148b69ae901f021b22450 is 50, key is test_row_0/C:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:16,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742087_1263 (size=12051) 2024-12-10T00:15:16,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3e8610ca807148b69ae901f021b22450 2024-12-10T00:15:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6c8f9d15c52b429184cf501605d83c99 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99 2024-12-10T00:15:16,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99, entries=250, sequenceid=131, filesize=16.4 K 2024-12-10T00:15:16,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4f871e73fcb74991b9cd67c0fe3ca08e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e 2024-12-10T00:15:16,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T00:15:16,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3e8610ca807148b69ae901f021b22450 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450 2024-12-10T00:15:16,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450, entries=150, sequenceid=131, filesize=11.8 K 2024-12-10T00:15:16,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for de1b03cb10a30c0ceaa7548777a2855e in 939ms, sequenceid=131, compaction requested=true 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:16,043 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:16,043 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:16,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:16,045 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:16,045 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:16,045 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:16,046 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/86c72046315142ed85bc4ad00bc08dd1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.4 K 2024-12-10T00:15:16,046 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:16,046 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:16,046 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:16,046 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/719570579eab457a85c5cccd86d72c00, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=40.0 K 2024-12-10T00:15:16,047 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c72046315142ed85bc4ad00bc08dd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733789713204 2024-12-10T00:15:16,047 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719570579eab457a85c5cccd86d72c00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733789713204 2024-12-10T00:15:16,048 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4df617146b48d7863ee9aa305e6686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733789713286 2024-12-10T00:15:16,048 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7c4bd1f2e142d0ab54a68e7c1706ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733789713286 2024-12-10T00:15:16,048 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8f9d15c52b429184cf501605d83c99, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:16,048 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f871e73fcb74991b9cd67c0fe3ca08e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:16,075 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:16,075 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/371a3dbb15a8466fb8908cd2d42c475f is 50, key is test_row_0/B:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:16,086 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:16,087 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e is 50, key is test_row_0/A:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:16,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742088_1264 (size=12359) 2024-12-10T00:15:16,125 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/371a3dbb15a8466fb8908cd2d42c475f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/371a3dbb15a8466fb8908cd2d42c475f 2024-12-10T00:15:16,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T00:15:16,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:16,132 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:16,137 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 371a3dbb15a8466fb8908cd2d42c475f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:16,137 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:16,137 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789716043; duration=0sec 2024-12-10T00:15:16,137 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:16,137 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:16,138 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:16,139 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:16,139 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:16,139 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:16,139 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/828b795fed374a0bafa4025216769ed9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=47.1 K 2024-12-10T00:15:16,140 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 828b795fed374a0bafa4025216769ed9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733789712547 2024-12-10T00:15:16,141 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f15dbb807834873b5f25d38ce010527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733789713204 2024-12-10T00:15:16,141 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 04066a18bebe4ed3ac23c8dd1d9e3ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733789713286 2024-12-10T00:15:16,142 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8610ca807148b69ae901f021b22450, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:16,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742089_1265 (size=12359) 2024-12-10T00:15:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6536dd1d03f04d5593b3d9e9abe2d1d1 is 50, key is test_row_0/A:col10/1733789715126/Put/seqid=0 2024-12-10T00:15:16,172 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e 2024-12-10T00:15:16,179 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 0fe671ddf37f4a6f9eec3a9b7f1c0a4e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:16,179 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:16,179 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789716043; duration=0sec 2024-12-10T00:15:16,179 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:16,179 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:16,182 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:16,183 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/cfdb1d5c05e04014ae4537d815ac04b9 is 50, key is test_row_0/C:col10/1733789714458/Put/seqid=0 2024-12-10T00:15:16,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742090_1266 (size=12151) 2024-12-10T00:15:16,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742091_1267 (size=12325) 2024-12-10T00:15:16,235 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/cfdb1d5c05e04014ae4537d815ac04b9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cfdb1d5c05e04014ae4537d815ac04b9 2024-12-10T00:15:16,243 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into cfdb1d5c05e04014ae4537d815ac04b9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:16,243 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:16,243 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=12, startTime=1733789716043; duration=0sec 2024-12-10T00:15:16,245 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:16,245 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:16,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:16,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789776310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789776314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789776419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789776419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,607 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6536dd1d03f04d5593b3d9e9abe2d1d1 2024-12-10T00:15:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4d1e4ebaae4a494887ef295eb4a4b3f9 is 50, key is test_row_0/B:col10/1733789715126/Put/seqid=0 2024-12-10T00:15:16,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742092_1268 (size=12151) 2024-12-10T00:15:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789776629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789776629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:16,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789776939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789776939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,031 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4d1e4ebaae4a494887ef295eb4a4b3f9 2024-12-10T00:15:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/d1cc48dcfc054085bdcd543dc2dea81e is 50, key is test_row_0/C:col10/1733789715126/Put/seqid=0 2024-12-10T00:15:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742093_1269 (size=12151) 2024-12-10T00:15:17,046 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/d1cc48dcfc054085bdcd543dc2dea81e 2024-12-10T00:15:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6536dd1d03f04d5593b3d9e9abe2d1d1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1 2024-12-10T00:15:17,059 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1, entries=150, sequenceid=148, filesize=11.9 K 2024-12-10T00:15:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4d1e4ebaae4a494887ef295eb4a4b3f9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9 2024-12-10T00:15:17,065 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9, entries=150, sequenceid=148, filesize=11.9 K 2024-12-10T00:15:17,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/d1cc48dcfc054085bdcd543dc2dea81e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e 2024-12-10T00:15:17,071 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e, entries=150, sequenceid=148, filesize=11.9 K 2024-12-10T00:15:17,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for de1b03cb10a30c0ceaa7548777a2855e in 940ms, sequenceid=148, compaction requested=false 2024-12-10T00:15:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-10T00:15:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-10T00:15:17,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-10T00:15:17,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-12-10T00:15:17,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4070 sec 2024-12-10T00:15:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:17,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:17,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6a83283a0d184346ad824e5e52b5b9a0 is 50, key is test_row_0/A:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742094_1270 (size=16931) 2024-12-10T00:15:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789777474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789777477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789777580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789777589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789777789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789777800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T00:15:17,829 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:17,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-10T00:15:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:17,831 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:17,832 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:17,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:17,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6a83283a0d184346ad824e5e52b5b9a0 2024-12-10T00:15:17,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/199a307ea18f45978a94edd973c9dfee is 50, key is test_row_0/B:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:17,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742095_1271 (size=12151) 2024-12-10T00:15:17,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/199a307ea18f45978a94edd973c9dfee 2024-12-10T00:15:17,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/19313dadcf2449eea091300ab91b6471 is 50, key is test_row_0/C:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:17,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742096_1272 (size=12151) 2024-12-10T00:15:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:17,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T00:15:17,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:17,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:17,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:17,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789778099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789778110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T00:15:18,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:18,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:18,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T00:15:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:18,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/19313dadcf2449eea091300ab91b6471 2024-12-10T00:15:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/6a83283a0d184346ad824e5e52b5b9a0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0 2024-12-10T00:15:18,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0, entries=250, sequenceid=168, filesize=16.5 K 2024-12-10T00:15:18,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/199a307ea18f45978a94edd973c9dfee as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee 2024-12-10T00:15:18,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee, entries=150, sequenceid=168, filesize=11.9 K 2024-12-10T00:15:18,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/19313dadcf2449eea091300ab91b6471 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471 2024-12-10T00:15:18,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471, entries=150, sequenceid=168, filesize=11.9 K 2024-12-10T00:15:18,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for de1b03cb10a30c0ceaa7548777a2855e in 899ms, sequenceid=168, compaction requested=true 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:18,351 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:18,351 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:18,352 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:18,352 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:18,352 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:18,352 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:18,353 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,353 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,353 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=40.5 K 2024-12-10T00:15:18,353 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/371a3dbb15a8466fb8908cd2d42c475f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.8 K 2024-12-10T00:15:18,353 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 371a3dbb15a8466fb8908cd2d42c475f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:18,353 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe671ddf37f4a6f9eec3a9b7f1c0a4e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:18,353 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6536dd1d03f04d5593b3d9e9abe2d1d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733789715122 2024-12-10T00:15:18,353 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1e4ebaae4a494887ef295eb4a4b3f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733789715122 2024-12-10T00:15:18,354 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 199a307ea18f45978a94edd973c9dfee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716312 2024-12-10T00:15:18,354 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a83283a0d184346ad824e5e52b5b9a0, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716287 2024-12-10T00:15:18,367 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:18,368 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5edb3303e2904d9d9a4daab8b14e19b6 is 50, key is test_row_0/A:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:18,371 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:18,372 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/a9c4429f4e9b4f0595cbce597160c165 is 50, key is test_row_0/B:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:18,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742097_1273 (size=12561) 2024-12-10T00:15:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742098_1274 (size=12561) 2024-12-10T00:15:18,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,443 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:18,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/b525217d32b9419fb78905d8f8ea4227 is 50, key is test_row_0/A:col10/1733789717464/Put/seqid=0 2024-12-10T00:15:18,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742099_1275 (size=12151) 2024-12-10T00:15:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789778638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789778642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789778750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789778750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,787 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5edb3303e2904d9d9a4daab8b14e19b6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5edb3303e2904d9d9a4daab8b14e19b6 2024-12-10T00:15:18,789 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/a9c4429f4e9b4f0595cbce597160c165 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/a9c4429f4e9b4f0595cbce597160c165 2024-12-10T00:15:18,803 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 5edb3303e2904d9d9a4daab8b14e19b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:18,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:18,803 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789718351; duration=0sec 2024-12-10T00:15:18,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:18,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:18,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:18,804 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into a9c4429f4e9b4f0595cbce597160c165(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:18,804 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:18,804 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789718351; duration=0sec 2024-12-10T00:15:18,804 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:18,804 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:18,804 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:18,804 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:18,804 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:18,804 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cfdb1d5c05e04014ae4537d815ac04b9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=35.8 K 2024-12-10T00:15:18,805 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfdb1d5c05e04014ae4537d815ac04b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733789714458 2024-12-10T00:15:18,805 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1cc48dcfc054085bdcd543dc2dea81e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733789715122 2024-12-10T00:15:18,806 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19313dadcf2449eea091300ab91b6471, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716312 2024-12-10T00:15:18,815 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:18,815 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3b22586d1d3340f18768c110e74eb718 is 50, key is test_row_0/C:col10/1733789716312/Put/seqid=0 2024-12-10T00:15:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742100_1276 (size=12527) 2024-12-10T00:15:18,850 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3b22586d1d3340f18768c110e74eb718 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3b22586d1d3340f18768c110e74eb718 2024-12-10T00:15:18,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/b525217d32b9419fb78905d8f8ea4227 2024-12-10T00:15:18,861 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 3b22586d1d3340f18768c110e74eb718(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:18,861 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:18,861 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=13, startTime=1733789718351; duration=0sec 2024-12-10T00:15:18,861 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:18,861 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:18,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/62f2a06f83204e4e98240d93a83a6bf9 is 50, key is test_row_0/B:col10/1733789717464/Put/seqid=0 2024-12-10T00:15:18,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742101_1277 (size=12151) 2024-12-10T00:15:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789778960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789778960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:19,269 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/62f2a06f83204e4e98240d93a83a6bf9 2024-12-10T00:15:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789779270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789779270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/14debc7cb4c6481590d6e78a2c012663 is 50, key is test_row_0/C:col10/1733789717464/Put/seqid=0 2024-12-10T00:15:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742102_1278 (size=12151) 2024-12-10T00:15:19,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/14debc7cb4c6481590d6e78a2c012663 2024-12-10T00:15:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/b525217d32b9419fb78905d8f8ea4227 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227 2024-12-10T00:15:19,291 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227, entries=150, sequenceid=183, filesize=11.9 K 2024-12-10T00:15:19,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/62f2a06f83204e4e98240d93a83a6bf9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9 2024-12-10T00:15:19,296 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9, entries=150, sequenceid=183, filesize=11.9 K 2024-12-10T00:15:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/14debc7cb4c6481590d6e78a2c012663 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663 2024-12-10T00:15:19,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663, entries=150, sequenceid=183, filesize=11.9 K 2024-12-10T00:15:19,302 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for de1b03cb10a30c0ceaa7548777a2855e in 858ms, sequenceid=183, compaction requested=false 2024-12-10T00:15:19,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:19,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:19,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-10T00:15:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-10T00:15:19,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T00:15:19,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4710 sec 2024-12-10T00:15:19,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4750 sec 2024-12-10T00:15:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:19,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/15a05c0616d84392993da669569a6859 is 50, key is test_row_0/A:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742103_1279 (size=12151) 2024-12-10T00:15:19,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/15a05c0616d84392993da669569a6859 2024-12-10T00:15:19,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789779700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789779707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789779709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/21b08f14abf44349be6b32f8229eda50 is 50, key is test_row_0/B:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742104_1280 (size=12151) 2024-12-10T00:15:19,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/21b08f14abf44349be6b32f8229eda50 2024-12-10T00:15:19,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/8551e5d1cb1147a2bf1761eec3d27b3d is 50, key is test_row_0/C:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742105_1281 (size=12151) 2024-12-10T00:15:19,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/8551e5d1cb1147a2bf1761eec3d27b3d 2024-12-10T00:15:19,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/15a05c0616d84392993da669569a6859 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859 2024-12-10T00:15:19,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859, entries=150, sequenceid=206, filesize=11.9 K 2024-12-10T00:15:19,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/21b08f14abf44349be6b32f8229eda50 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50 2024-12-10T00:15:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789779780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789779781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50, entries=150, sequenceid=206, filesize=11.9 K 2024-12-10T00:15:19,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/8551e5d1cb1147a2bf1761eec3d27b3d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d 2024-12-10T00:15:19,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d, entries=150, sequenceid=206, filesize=11.9 K 2024-12-10T00:15:19,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for de1b03cb10a30c0ceaa7548777a2855e in 115ms, sequenceid=206, compaction requested=true 2024-12-10T00:15:19,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:19,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:19,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:19,799 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:19,799 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:19,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:19,801 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:19,801 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:19,801 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:19,801 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5edb3303e2904d9d9a4daab8b14e19b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.0 K 2024-12-10T00:15:19,801 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:19,802 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:19,802 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:19,802 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/a9c4429f4e9b4f0595cbce597160c165, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.0 K 2024-12-10T00:15:19,802 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5edb3303e2904d9d9a4daab8b14e19b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716312 2024-12-10T00:15:19,802 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c4429f4e9b4f0595cbce597160c165, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716312 2024-12-10T00:15:19,803 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f2a06f83204e4e98240d93a83a6bf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733789717464 2024-12-10T00:15:19,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b525217d32b9419fb78905d8f8ea4227, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733789717464 2024-12-10T00:15:19,803 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b08f14abf44349be6b32f8229eda50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:19,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a05c0616d84392993da669569a6859, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:19,813 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:19,814 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/cff43c31f6f1465fb1290dfeed2245eb is 50, key is test_row_0/B:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:19,818 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5f81e62424764e04932e9b340f2ebb68 is 50, key is test_row_0/A:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:19,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:15:19,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:19,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/085dfe6fd2884327a550fe1e62007ff5 is 50, key is test_row_0/A:col10/1733789719823/Put/seqid=0 2024-12-10T00:15:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742106_1282 (size=12663) 2024-12-10T00:15:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742107_1283 (size=12663) 2024-12-10T00:15:19,871 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/5f81e62424764e04932e9b340f2ebb68 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5f81e62424764e04932e9b340f2ebb68 2024-12-10T00:15:19,876 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 5f81e62424764e04932e9b340f2ebb68(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:19,877 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:19,877 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789719798; duration=0sec 2024-12-10T00:15:19,879 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:19,879 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:19,879 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:19,880 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:19,880 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:19,880 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:19,880 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3b22586d1d3340f18768c110e74eb718, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.0 K 2024-12-10T00:15:19,881 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b22586d1d3340f18768c110e74eb718, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733789716312 2024-12-10T00:15:19,881 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14debc7cb4c6481590d6e78a2c012663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733789717464 2024-12-10T00:15:19,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742108_1284 (size=12151) 2024-12-10T00:15:19,883 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8551e5d1cb1147a2bf1761eec3d27b3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:19,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789779876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789779882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789779889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:19,909 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#241 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:19,910 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/8a40291ed88443d898cc9cd5d2039a17 is 50, key is test_row_0/C:col10/1733789719682/Put/seqid=0 2024-12-10T00:15:19,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742109_1285 (size=12629) 2024-12-10T00:15:19,955 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/8a40291ed88443d898cc9cd5d2039a17 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8a40291ed88443d898cc9cd5d2039a17 2024-12-10T00:15:19,969 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 8a40291ed88443d898cc9cd5d2039a17(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:19,969 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:19,969 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=13, startTime=1733789719800; duration=0sec 2024-12-10T00:15:19,969 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:19,969 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T00:15:19,980 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:19,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-10T00:15:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:19,983 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:19,984 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:19,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:20,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789779999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789780000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789780000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:20,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789780210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789780210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789780210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,267 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/cff43c31f6f1465fb1290dfeed2245eb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/cff43c31f6f1465fb1290dfeed2245eb 2024-12-10T00:15:20,271 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into cff43c31f6f1465fb1290dfeed2245eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:20,271 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:20,271 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789719799; duration=0sec 2024-12-10T00:15:20,271 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:20,271 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:20,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/085dfe6fd2884327a550fe1e62007ff5 2024-12-10T00:15:20,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/1b2c0cbd27ff4c57894e01339a679a4d is 50, key is test_row_0/B:col10/1733789719823/Put/seqid=0 2024-12-10T00:15:20,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742110_1286 (size=12151) 2024-12-10T00:15:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:20,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789780519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789780519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789780519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:20,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/1b2c0cbd27ff4c57894e01339a679a4d 2024-12-10T00:15:20,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/c9a466c3bea04396b9d51a66aa004265 is 50, key is test_row_0/C:col10/1733789719823/Put/seqid=0 2024-12-10T00:15:20,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742111_1287 (size=12151) 2024-12-10T00:15:20,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789780800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789780800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:20,802 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,803 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:20,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:20,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:21,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789781030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789781040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789781040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:21,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:21,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:21,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/c9a466c3bea04396b9d51a66aa004265 2024-12-10T00:15:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:21,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/085dfe6fd2884327a550fe1e62007ff5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5 2024-12-10T00:15:21,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T00:15:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/1b2c0cbd27ff4c57894e01339a679a4d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d 2024-12-10T00:15:21,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T00:15:21,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/c9a466c3bea04396b9d51a66aa004265 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265 2024-12-10T00:15:21,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T00:15:21,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for de1b03cb10a30c0ceaa7548777a2855e in 1323ms, sequenceid=221, compaction requested=false 2024-12-10T00:15:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:21,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:21,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:21,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/688806ba6198467bbf014daff8318ecc is 50, key is test_row_0/A:col10/1733789719880/Put/seqid=0 2024-12-10T00:15:21,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742112_1288 (size=12151) 2024-12-10T00:15:21,618 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/688806ba6198467bbf014daff8318ecc 2024-12-10T00:15:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4ceeb07efeef41a68dd2113ae3a5a638 is 50, key is test_row_0/B:col10/1733789719880/Put/seqid=0 2024-12-10T00:15:21,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742113_1289 (size=12151) 2024-12-10T00:15:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4ceeb07efeef41a68dd2113ae3a5a638 2024-12-10T00:15:21,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/37e4c8376a8c42bf8bc0f333ded4fdf2 is 50, key is test_row_0/C:col10/1733789719880/Put/seqid=0 2024-12-10T00:15:21,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742114_1290 (size=12151) 2024-12-10T00:15:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:22,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,063 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/37e4c8376a8c42bf8bc0f333ded4fdf2 2024-12-10T00:15:22,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/688806ba6198467bbf014daff8318ecc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc 2024-12-10T00:15:22,071 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T00:15:22,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/4ceeb07efeef41a68dd2113ae3a5a638 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638 2024-12-10T00:15:22,075 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T00:15:22,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/37e4c8376a8c42bf8bc0f333ded4fdf2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2 2024-12-10T00:15:22,079 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T00:15:22,085 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=33.54 KB/34350 for de1b03cb10a30c0ceaa7548777a2855e in 876ms, sequenceid=241, compaction requested=true 2024-12-10T00:15:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-10T00:15:22,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:22,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-10T00:15:22,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1030 sec 2024-12-10T00:15:22,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d69a27940f9c4d86bf69d56558cabde6 is 50, key is test_row_0/A:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:22,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1110 sec 2024-12-10T00:15:22,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742115_1291 (size=12151) 2024-12-10T00:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T00:15:22,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:22,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-10T00:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:22,132 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:22,133 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:22,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:22,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789722114, max timestamp=1733789722152Num uniques 859; 2024-12-10T00:15:22,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:22,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789722114, max timestamp=1733789722152Num uniques 859; 2024-12-10T00:15:22,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:22,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789722114, max timestamp=1733789722152Num uniques 859; 2024-12-10T00:15:22,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T00:15:22,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789782186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789782188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789782197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:22,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:22,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789782300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789782300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789782309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:22,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d69a27940f9c4d86bf69d56558cabde6 2024-12-10T00:15:22,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/d2b956e18f574d23bbc45184b0b48720 is 50, key is test_row_0/B:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742116_1292 (size=12151) 2024-12-10T00:15:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789782510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789782510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789782519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789782819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789782819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789782830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789782830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789782830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,831 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:22,831 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:22,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:22,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:22,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/d2b956e18f574d23bbc45184b0b48720 2024-12-10T00:15:22,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/2de54e4e15cd4c91bf4484d2fcaf9192 is 50, key is test_row_0/C:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:22,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742117_1293 (size=12151) 2024-12-10T00:15:23,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:23,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/2de54e4e15cd4c91bf4484d2fcaf9192 2024-12-10T00:15:23,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d69a27940f9c4d86bf69d56558cabde6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6 2024-12-10T00:15:23,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6, entries=150, sequenceid=252, filesize=11.9 K 2024-12-10T00:15:23,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789783330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789783330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:23,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/d2b956e18f574d23bbc45184b0b48720 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:23,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720, entries=150, sequenceid=252, filesize=11.9 K 2024-12-10T00:15:23,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/2de54e4e15cd4c91bf4484d2fcaf9192 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192 2024-12-10T00:15:23,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192, entries=150, sequenceid=252, filesize=11.9 K 2024-12-10T00:15:23,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for de1b03cb10a30c0ceaa7548777a2855e in 1255ms, sequenceid=252, compaction requested=true 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:23,341 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:23,341 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:23,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:23,342 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:23,342 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:23,342 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:23,342 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:23,342 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,342 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,342 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5f81e62424764e04932e9b340f2ebb68, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=48.0 K 2024-12-10T00:15:23,342 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/cff43c31f6f1465fb1290dfeed2245eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=48.0 K 2024-12-10T00:15:23,343 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f81e62424764e04932e9b340f2ebb68, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:23,343 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cff43c31f6f1465fb1290dfeed2245eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:23,343 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085dfe6fd2884327a550fe1e62007ff5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733789719823 2024-12-10T00:15:23,343 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2c0cbd27ff4c57894e01339a679a4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733789719823 2024-12-10T00:15:23,344 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 688806ba6198467bbf014daff8318ecc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789719852 2024-12-10T00:15:23,344 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ceeb07efeef41a68dd2113ae3a5a638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789719852 2024-12-10T00:15:23,344 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69a27940f9c4d86bf69d56558cabde6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:23,344 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b956e18f574d23bbc45184b0b48720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:23,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:23,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/7f7bca7329274b898d6750d48b72ebd4 is 50, key is test_row_0/A:col10/1733789722185/Put/seqid=0 2024-12-10T00:15:23,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,371 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:23,372 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/bdf62cc879d848268ed5110ff0f4000a is 50, key is test_row_0/B:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:23,375 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:23,375 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/f59f211a401a40848be39853785dd09d is 50, key is test_row_0/A:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:23,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742118_1294 (size=14741) 2024-12-10T00:15:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742119_1295 (size=12799) 2024-12-10T00:15:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742120_1296 (size=12799) 2024-12-10T00:15:23,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789783403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:23,416 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/f59f211a401a40848be39853785dd09d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f59f211a401a40848be39853785dd09d 2024-12-10T00:15:23,422 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into f59f211a401a40848be39853785dd09d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:23,422 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:23,422 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=12, startTime=1733789723341; duration=0sec 2024-12-10T00:15:23,423 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:23,423 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:23,423 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:23,424 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:23,424 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:23,424 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,424 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8a40291ed88443d898cc9cd5d2039a17, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=47.9 K 2024-12-10T00:15:23,425 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a40291ed88443d898cc9cd5d2039a17, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733789718637 2024-12-10T00:15:23,425 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a466c3bea04396b9d51a66aa004265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733789719823 2024-12-10T00:15:23,426 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e4c8376a8c42bf8bc0f333ded4fdf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789719852 2024-12-10T00:15:23,427 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de54e4e15cd4c91bf4484d2fcaf9192, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:23,434 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:23,434 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/170b18d980c2496688b541242bdfa7cf is 50, key is test_row_0/C:col10/1733789722058/Put/seqid=0 2024-12-10T00:15:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742121_1297 (size=12765) 2024-12-10T00:15:23,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789783509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:23,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789783719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:23,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/7f7bca7329274b898d6750d48b72ebd4 2024-12-10T00:15:23,799 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/bdf62cc879d848268ed5110ff0f4000a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/bdf62cc879d848268ed5110ff0f4000a 2024-12-10T00:15:23,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7aa07f0c0a4e432dbb0cd2106d35468e is 50, key is test_row_0/B:col10/1733789722185/Put/seqid=0 2024-12-10T00:15:23,804 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into bdf62cc879d848268ed5110ff0f4000a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:23,804 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:23,804 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=12, startTime=1733789723341; duration=0sec 2024-12-10T00:15:23,805 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:23,805 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:23,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742122_1298 (size=12301) 2024-12-10T00:15:23,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,843 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/170b18d980c2496688b541242bdfa7cf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/170b18d980c2496688b541242bdfa7cf 2024-12-10T00:15:23,847 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 170b18d980c2496688b541242bdfa7cf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:23,847 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:23,847 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=12, startTime=1733789723341; duration=0sec 2024-12-10T00:15:23,847 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:23,847 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:23,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:23,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789784029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:24,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:24,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:24,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7aa07f0c0a4e432dbb0cd2106d35468e 2024-12-10T00:15:24,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/cb76121459ce40baacc2347dbf4a9889 is 50, key is test_row_0/C:col10/1733789722185/Put/seqid=0 2024-12-10T00:15:24,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742123_1299 (size=12301) 2024-12-10T00:15:24,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/cb76121459ce40baacc2347dbf4a9889 2024-12-10T00:15:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/7f7bca7329274b898d6750d48b72ebd4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4 2024-12-10T00:15:24,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4, entries=200, sequenceid=288, filesize=14.4 K 2024-12-10T00:15:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7aa07f0c0a4e432dbb0cd2106d35468e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e 2024-12-10T00:15:24,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e, entries=150, sequenceid=288, filesize=12.0 K 2024-12-10T00:15:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/cb76121459ce40baacc2347dbf4a9889 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889 2024-12-10T00:15:24,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889, entries=150, sequenceid=288, filesize=12.0 K 2024-12-10T00:15:24,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for de1b03cb10a30c0ceaa7548777a2855e in 910ms, sequenceid=288, compaction requested=false 2024-12-10T00:15:24,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:24,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T00:15:24,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:24,280 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:24,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:24,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:24,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:24,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/58094d9d81be4067af6e82c74a114d44 is 50, key is test_row_1/A:col10/1733789723362/Put/seqid=0 2024-12-10T00:15:24,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742124_1300 (size=9857) 2024-12-10T00:15:24,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/58094d9d81be4067af6e82c74a114d44 2024-12-10T00:15:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/e46e5e634ab94d359b846e5a2644f2dc is 50, key is test_row_1/B:col10/1733789723362/Put/seqid=0 2024-12-10T00:15:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742125_1301 (size=9857) 2024-12-10T00:15:24,310 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/e46e5e634ab94d359b846e5a2644f2dc 2024-12-10T00:15:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/52238b915be245d99e6efacd5e2d36f5 is 50, key is test_row_1/C:col10/1733789723362/Put/seqid=0 2024-12-10T00:15:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742126_1302 (size=9857) 2024-12-10T00:15:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:24,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789724362, max timestamp=1733789724435Num uniques 859; 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789724362, max timestamp=1733789724435Num uniques 859; 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789724362, max timestamp=1733789724435Num uniques 859; 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:24,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789724458, max timestamp=1733789724481Num uniques 1018; 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789724458, max timestamp=1733789724481Num uniques 1018; 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789724458, max timestamp=1733789724481Num uniques 1018; 2024-12-10T00:15:24,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T00:15:24,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789784533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789784535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789784540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789784640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789784649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,731 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/52238b915be245d99e6efacd5e2d36f5 2024-12-10T00:15:24,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/58094d9d81be4067af6e82c74a114d44 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44 2024-12-10T00:15:24,753 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44, entries=100, sequenceid=301, filesize=9.6 K 2024-12-10T00:15:24,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/e46e5e634ab94d359b846e5a2644f2dc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc 2024-12-10T00:15:24,760 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc, entries=100, sequenceid=301, filesize=9.6 K 2024-12-10T00:15:24,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/52238b915be245d99e6efacd5e2d36f5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5 2024-12-10T00:15:24,766 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5, entries=100, sequenceid=301, filesize=9.6 K 2024-12-10T00:15:24,767 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for de1b03cb10a30c0ceaa7548777a2855e in 487ms, sequenceid=301, compaction requested=true 2024-12-10T00:15:24,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:24,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-10T00:15:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-10T00:15:24,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-10T00:15:24,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-12-10T00:15:24,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.6420 sec 2024-12-10T00:15:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:24,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-12-10T00:15:24,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:24,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:24,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:24,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0058029d0cc6466e806d030b4ee7542c is 50, key is test_row_0/A:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:24,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742127_1303 (size=14741) 2024-12-10T00:15:24,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789784868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789784872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789784979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789784979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789785190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789785190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0058029d0cc6466e806d030b4ee7542c 2024-12-10T00:15:25,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7f51ae0deeb640f990cc8ff0d7cbdda4 is 50, key is test_row_0/B:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742128_1304 (size=12301) 2024-12-10T00:15:25,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789785499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789785499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789785559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:25,562 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:25,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7f51ae0deeb640f990cc8ff0d7cbdda4 2024-12-10T00:15:25,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6f8a815625ad4197b0cb9069ac762f09 is 50, key is test_row_0/C:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:25,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742129_1305 (size=12301) 2024-12-10T00:15:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789786009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789786009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:26,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6f8a815625ad4197b0cb9069ac762f09 2024-12-10T00:15:26,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0058029d0cc6466e806d030b4ee7542c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c 2024-12-10T00:15:26,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c, entries=200, sequenceid=355, filesize=14.4 K 2024-12-10T00:15:26,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7f51ae0deeb640f990cc8ff0d7cbdda4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4 2024-12-10T00:15:26,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T00:15:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/6f8a815625ad4197b0cb9069ac762f09 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09 2024-12-10T00:15:26,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09, entries=150, sequenceid=355, filesize=12.0 K 2024-12-10T00:15:26,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for de1b03cb10a30c0ceaa7548777a2855e in 1292ms, sequenceid=355, compaction requested=true 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:26,144 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:26,144 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:26,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:26,146 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:26,146 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:26,146 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:26,146 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:26,146 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:26,146 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:26,146 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/bdf62cc879d848268ed5110ff0f4000a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=46.2 K 2024-12-10T00:15:26,146 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f59f211a401a40848be39853785dd09d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=50.9 K 2024-12-10T00:15:26,147 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59f211a401a40848be39853785dd09d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:26,147 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf62cc879d848268ed5110ff0f4000a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:26,147 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7bca7329274b898d6750d48b72ebd4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733789722183 2024-12-10T00:15:26,147 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa07f0c0a4e432dbb0cd2106d35468e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733789722185 2024-12-10T00:15:26,147 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58094d9d81be4067af6e82c74a114d44, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733789723362 2024-12-10T00:15:26,148 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e46e5e634ab94d359b846e5a2644f2dc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733789723362 2024-12-10T00:15:26,148 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0058029d0cc6466e806d030b4ee7542c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724514 2024-12-10T00:15:26,148 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f51ae0deeb640f990cc8ff0d7cbdda4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724521 2024-12-10T00:15:26,159 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:26,159 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/37286ad7010c4400b4e22d3603210d6f is 50, key is test_row_0/B:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:26,162 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:26,162 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/71773fc3b4c54673bd5c255c62b2247a is 50, key is test_row_0/A:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:26,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742130_1306 (size=13085) 2024-12-10T00:15:26,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742131_1307 (size=13085) 2024-12-10T00:15:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T00:15:26,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:26,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-10T00:15:26,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:26,315 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:26,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T00:15:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T00:15:26,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-10T00:15:26,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:26,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:26,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a96889b9261b4275ab0665d12c2c3ef0 is 50, key is test_row_0/A:col10/1733789724870/Put/seqid=0 2024-12-10T00:15:26,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742132_1308 (size=12301) 2024-12-10T00:15:26,573 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/37286ad7010c4400b4e22d3603210d6f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/37286ad7010c4400b4e22d3603210d6f 2024-12-10T00:15:26,580 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 37286ad7010c4400b4e22d3603210d6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:26,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:26,581 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=12, startTime=1733789726144; duration=0sec 2024-12-10T00:15:26,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:26,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:26,581 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:26,582 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:26,582 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:26,582 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:26,582 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/170b18d980c2496688b541242bdfa7cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=46.1 K 2024-12-10T00:15:26,583 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 170b18d980c2496688b541242bdfa7cf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733789722058 2024-12-10T00:15:26,583 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cb76121459ce40baacc2347dbf4a9889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733789722185 2024-12-10T00:15:26,584 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 52238b915be245d99e6efacd5e2d36f5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733789723362 2024-12-10T00:15:26,584 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f8a815625ad4197b0cb9069ac762f09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724521 2024-12-10T00:15:26,587 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/71773fc3b4c54673bd5c255c62b2247a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/71773fc3b4c54673bd5c255c62b2247a 2024-12-10T00:15:26,591 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 71773fc3b4c54673bd5c255c62b2247a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:26,591 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:26,591 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=12, startTime=1733789726144; duration=0sec 2024-12-10T00:15:26,591 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:26,591 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:26,591 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:26,592 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/fcfca2f168004992a54ec408ec029b5d is 50, key is test_row_0/C:col10/1733789724533/Put/seqid=0 2024-12-10T00:15:26,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742133_1309 (size=13051) 2024-12-10T00:15:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T00:15:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:26,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:26,882 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a96889b9261b4275ab0665d12c2c3ef0 2024-12-10T00:15:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/914b6d93f1c04ea89842f1c1b523965e is 50, key is test_row_0/B:col10/1733789724870/Put/seqid=0 2024-12-10T00:15:26,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742134_1310 (size=12301) 2024-12-10T00:15:26,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789726867, max timestamp=1733789726899Num uniques 859; 2024-12-10T00:15:26,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:26,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789726867, max timestamp=1733789726899Num uniques 859; 2024-12-10T00:15:26,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:26,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789726867, max timestamp=1733789726899Num uniques 859; 2024-12-10T00:15:26,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:15:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789786931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789726924, max timestamp=1733789726933Num uniques 1018; 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789726924, max timestamp=1733789726933Num uniques 1018; 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733789726924, max timestamp=1733789726933Num uniques 1018; 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:15:26,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T00:15:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T00:15:26,999 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/fcfca2f168004992a54ec408ec029b5d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/fcfca2f168004992a54ec408ec029b5d 2024-12-10T00:15:27,013 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into fcfca2f168004992a54ec408ec029b5d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:27,013 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:27,013 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=12, startTime=1733789726144; duration=0sec 2024-12-10T00:15:27,013 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:27,013 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789726983, max timestamp=1733789727017Num uniques 1027; 2024-12-10T00:15:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T00:15:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789726983, max timestamp=1733789727017Num uniques 1027; 2024-12-10T00:15:27,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T00:15:27,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789726983, max timestamp=1733789727017Num uniques 1027; 2024-12-10T00:15:27,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789727034, max timestamp=1733789727048Num uniques 1036; 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8512, totalHeapSize=19584, min timestamp=1733789727034, max timestamp=1733789727049Num uniques 1036; 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47442, new segment data size=8512, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789727034, max timestamp=1733789727048Num uniques 1036; 2024-12-10T00:15:27,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:27,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789787072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733789727061, max timestamp=1733789727075Num uniques 1045; 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733789727061, max timestamp=1733789727075Num uniques 1045; 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733789727061, max timestamp=1733789727075Num uniques 1045; 2024-12-10T00:15:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-10T00:15:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789787101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789787102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789787104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789787190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789787210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789787210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789787211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,294 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/914b6d93f1c04ea89842f1c1b523965e 2024-12-10T00:15:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/157a40e9a3f04c6e94d4f7b31fe2ce04 is 50, key is test_row_0/C:col10/1733789724870/Put/seqid=0 2024-12-10T00:15:27,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742135_1311 (size=12301) 2024-12-10T00:15:27,319 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/157a40e9a3f04c6e94d4f7b31fe2ce04 2024-12-10T00:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a96889b9261b4275ab0665d12c2c3ef0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0 2024-12-10T00:15:27,328 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:15:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/914b6d93f1c04ea89842f1c1b523965e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e 2024-12-10T00:15:27,336 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:15:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/157a40e9a3f04c6e94d4f7b31fe2ce04 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04 2024-12-10T00:15:27,341 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:15:27,342 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for de1b03cb10a30c0ceaa7548777a2855e in 874ms, sequenceid=364, compaction requested=false 2024-12-10T00:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-10T00:15:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-10T00:15:27,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-10T00:15:27,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0290 sec 2024-12-10T00:15:27,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0370 sec 2024-12-10T00:15:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:27,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-12-10T00:15:27,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:15:27,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a517372f1735457f8f7274ab28e12145 is 50, key is test_row_0/A:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:27,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742136_1312 (size=12301) 2024-12-10T00:15:27,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789787420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789787420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789787420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789787420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T00:15:27,460 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:27,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-10T00:15:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T00:15:27,462 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:27,462 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:27,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:27,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789787529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T00:15:27,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44730 deadline: 1733789787580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,584 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789787730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789787730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789787730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789787739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:27,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:27,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:27,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T00:15:27,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a517372f1735457f8f7274ab28e12145 2024-12-10T00:15:27,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/de3777810f544defa1b0fcc351dbc1fb is 50, key is test_row_0/B:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:27,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742137_1313 (size=12301) 2024-12-10T00:15:27,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:27,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:27,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:27,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789788049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T00:15:28,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/de3777810f544defa1b0fcc351dbc1fb 2024-12-10T00:15:28,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789788242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789788243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789788243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/96ad8f324d514017bfc39cbb02af753e is 50, key is test_row_0/C:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742138_1314 (size=12301) 2024-12-10T00:15:28,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/96ad8f324d514017bfc39cbb02af753e 2024-12-10T00:15:28,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a517372f1735457f8f7274ab28e12145 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145 2024-12-10T00:15:28,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145, entries=150, sequenceid=474, filesize=12.0 K 2024-12-10T00:15:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/de3777810f544defa1b0fcc351dbc1fb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb 2024-12-10T00:15:28,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb, entries=150, sequenceid=474, filesize=12.0 K 2024-12-10T00:15:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/96ad8f324d514017bfc39cbb02af753e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e 2024-12-10T00:15:28,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e, entries=150, sequenceid=474, filesize=12.0 K 2024-12-10T00:15:28,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for de1b03cb10a30c0ceaa7548777a2855e in 876ms, sequenceid=474, compaction requested=true 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:28,278 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,278 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:28,280 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,280 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,280 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/37286ad7010c4400b4e22d3603210d6f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.8 K 2024-12-10T00:15:28,280 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/71773fc3b4c54673bd5c255c62b2247a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.8 K 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 37286ad7010c4400b4e22d3603210d6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724521 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71773fc3b4c54673bd5c255c62b2247a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724521 2024-12-10T00:15:28,280 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 914b6d93f1c04ea89842f1c1b523965e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789724866 2024-12-10T00:15:28,281 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96889b9261b4275ab0665d12c2c3ef0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789724866 2024-12-10T00:15:28,281 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting de3777810f544defa1b0fcc351dbc1fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,281 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a517372f1735457f8f7274ab28e12145, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,290 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,291 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/3f1d584a455b4db3a3edae367f9fece5 is 50, key is test_row_0/B:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:28,293 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,294 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/9369652ddea7412cb785ecf8783310dc is 50, key is test_row_0/A:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742139_1315 (size=13187) 2024-12-10T00:15:28,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742140_1316 (size=13187) 2024-12-10T00:15:28,304 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/3f1d584a455b4db3a3edae367f9fece5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/3f1d584a455b4db3a3edae367f9fece5 2024-12-10T00:15:28,312 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/9369652ddea7412cb785ecf8783310dc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/9369652ddea7412cb785ecf8783310dc 2024-12-10T00:15:28,315 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 3f1d584a455b4db3a3edae367f9fece5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:28,315 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,315 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789728278; duration=0sec 2024-12-10T00:15:28,315 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:28,315 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:28,315 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,317 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,317 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:28,317 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,317 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/fcfca2f168004992a54ec408ec029b5d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.8 K 2024-12-10T00:15:28,318 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fcfca2f168004992a54ec408ec029b5d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733789724521 2024-12-10T00:15:28,319 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 9369652ddea7412cb785ecf8783310dc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:28,319 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,319 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789728278; duration=0sec 2024-12-10T00:15:28,319 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,319 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:28,319 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 157a40e9a3f04c6e94d4f7b31fe2ce04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789724866 2024-12-10T00:15:28,320 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ad8f324d514017bfc39cbb02af753e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,329 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,329 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/2b55740d7f9f454f9d2d58770136f5c6 is 50, key is test_row_0/C:col10/1733789727098/Put/seqid=0 2024-12-10T00:15:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742141_1317 (size=13153) 2024-12-10T00:15:28,346 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/2b55740d7f9f454f9d2d58770136f5c6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2b55740d7f9f454f9d2d58770136f5c6 2024-12-10T00:15:28,352 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 2b55740d7f9f454f9d2d58770136f5c6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:28,352 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,352 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=13, startTime=1733789728278; duration=0sec 2024-12-10T00:15:28,352 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,352 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:28,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a5e7f586251d46eb8ccf7bd9c0b529e8 is 50, key is test_row_0/A:col10/1733789727412/Put/seqid=0 2024-12-10T00:15:28,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742142_1318 (size=9857) 2024-12-10T00:15:28,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a5e7f586251d46eb8ccf7bd9c0b529e8 2024-12-10T00:15:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/71dd05d5a9944e1ea8836da7e5b19960 is 50, key is test_row_0/B:col10/1733789727412/Put/seqid=0 2024-12-10T00:15:28,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742143_1319 (size=9857) 2024-12-10T00:15:28,444 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/71dd05d5a9944e1ea8836da7e5b19960 2024-12-10T00:15:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/e1c05c4a19cf4524a2ce2af481b029df is 50, key is test_row_0/C:col10/1733789727412/Put/seqid=0 2024-12-10T00:15:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742144_1320 (size=9857) 2024-12-10T00:15:28,466 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/e1c05c4a19cf4524a2ce2af481b029df 2024-12-10T00:15:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/a5e7f586251d46eb8ccf7bd9c0b529e8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8 2024-12-10T00:15:28,481 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8, entries=100, sequenceid=485, filesize=9.6 K 2024-12-10T00:15:28,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/71dd05d5a9944e1ea8836da7e5b19960 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960 2024-12-10T00:15:28,489 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960, entries=100, sequenceid=485, filesize=9.6 K 2024-12-10T00:15:28,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/e1c05c4a19cf4524a2ce2af481b029df as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df 2024-12-10T00:15:28,497 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df, entries=100, sequenceid=485, filesize=9.6 K 2024-12-10T00:15:28,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for de1b03cb10a30c0ceaa7548777a2855e in 120ms, sequenceid=485, compaction requested=false 2024-12-10T00:15:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-10T00:15:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-10T00:15:28,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-10T00:15:28,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-12-10T00:15:28,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0450 sec 2024-12-10T00:15:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:28,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:28,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0b6498c645c34340b16e00a9682e2975 is 50, key is test_row_0/A:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742145_1321 (size=14741) 2024-12-10T00:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-10T00:15:28,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:28,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:28,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0b6498c645c34340b16e00a9682e2975 2024-12-10T00:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-10T00:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:28,603 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:28,604 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:28,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:28,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/841766aa96054fb08651b5ce9bd056c7 is 50, key is test_row_0/B:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742146_1322 (size=12301) 2024-12-10T00:15:28,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/841766aa96054fb08651b5ce9bd056c7 2024-12-10T00:15:28,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4d203e9599e4408288710b61b30da58b is 50, key is test_row_0/C:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742147_1323 (size=12301) 2024-12-10T00:15:28,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4d203e9599e4408288710b61b30da58b 2024-12-10T00:15:28,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0b6498c645c34340b16e00a9682e2975 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975 2024-12-10T00:15:28,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975, entries=200, sequenceid=496, filesize=14.4 K 2024-12-10T00:15:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/841766aa96054fb08651b5ce9bd056c7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7 2024-12-10T00:15:28,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T00:15:28,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4d203e9599e4408288710b61b30da58b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b 2024-12-10T00:15:28,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b, entries=150, sequenceid=496, filesize=12.0 K 2024-12-10T00:15:28,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=114.05 KB/116790 for de1b03cb10a30c0ceaa7548777a2855e in 87ms, sequenceid=496, compaction requested=true 2024-12-10T00:15:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,671 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:28,672 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de1b03cb10a30c0ceaa7548777a2855e:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:28,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37785 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/A is initiating minor compaction (all files) 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/B is initiating minor compaction (all files) 2024-12-10T00:15:28,673 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/A in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,673 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/B in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,673 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/3f1d584a455b4db3a3edae367f9fece5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=34.5 K 2024-12-10T00:15:28,673 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/9369652ddea7412cb785ecf8783310dc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=36.9 K 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9369652ddea7412cb785ecf8783310dc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,673 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1d584a455b4db3a3edae367f9fece5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,674 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e7f586251d46eb8ccf7bd9c0b529e8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733789727412 2024-12-10T00:15:28,674 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 71dd05d5a9944e1ea8836da7e5b19960, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733789727412 2024-12-10T00:15:28,674 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6498c645c34340b16e00a9682e2975, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733789728572 2024-12-10T00:15:28,674 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 841766aa96054fb08651b5ce9bd056c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733789728572 2024-12-10T00:15:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:28,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-10T00:15:28,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:28,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:28,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:28,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:28,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0c5430ae6fed48d1a4c3fd271a1f5d8b is 50, key is test_row_0/A:col10/1733789728657/Put/seqid=0 2024-12-10T00:15:28,703 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#A#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,703 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/7aa7bf7a430c4373b9574c6afe8df38f is 50, key is test_row_0/A:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,706 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,707 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/71bc8655732047e79e1dea0043b49cdb is 50, key is test_row_0/B:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:28,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742148_1324 (size=14741) 2024-12-10T00:15:28,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742149_1325 (size=13289) 2024-12-10T00:15:28,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789788716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742150_1326 (size=13289) 2024-12-10T00:15:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,724 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/7aa7bf7a430c4373b9574c6afe8df38f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7aa7bf7a430c4373b9574c6afe8df38f 2024-12-10T00:15:28,733 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/71bc8655732047e79e1dea0043b49cdb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71bc8655732047e79e1dea0043b49cdb 2024-12-10T00:15:28,735 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/A of de1b03cb10a30c0ceaa7548777a2855e into 7aa7bf7a430c4373b9574c6afe8df38f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:28,735 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,735 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/A, priority=13, startTime=1733789728671; duration=0sec 2024-12-10T00:15:28,736 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:28,736 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:A 2024-12-10T00:15:28,736 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:28,737 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:28,737 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): de1b03cb10a30c0ceaa7548777a2855e/C is initiating minor compaction (all files) 2024-12-10T00:15:28,737 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de1b03cb10a30c0ceaa7548777a2855e/C in TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,737 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2b55740d7f9f454f9d2d58770136f5c6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp, totalSize=34.5 K 2024-12-10T00:15:28,737 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b55740d7f9f454f9d2d58770136f5c6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733789727098 2024-12-10T00:15:28,737 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c05c4a19cf4524a2ce2af481b029df, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733789727412 2024-12-10T00:15:28,738 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d203e9599e4408288710b61b30da58b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733789728572 2024-12-10T00:15:28,742 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/B of de1b03cb10a30c0ceaa7548777a2855e into 71bc8655732047e79e1dea0043b49cdb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:28,742 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:28,742 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/B, priority=13, startTime=1733789728671; duration=0sec 2024-12-10T00:15:28,743 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:28,743 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:B 2024-12-10T00:15:28,755 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de1b03cb10a30c0ceaa7548777a2855e#C#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:28,757 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/9c19995e1dbc4e61a317356713e7dadf is 50, key is test_row_0/C:col10/1733789728572/Put/seqid=0 2024-12-10T00:15:28,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742151_1327 (size=13255) 2024-12-10T00:15:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789788829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:28,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:28,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789789040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0c5430ae6fed48d1a4c3fd271a1f5d8b 2024-12-10T00:15:29,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/b886f9839ca24f499396794f0a1fe121 is 50, key is test_row_0/B:col10/1733789728657/Put/seqid=0 2024-12-10T00:15:29,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742152_1328 (size=12301) 2024-12-10T00:15:29,175 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/9c19995e1dbc4e61a317356713e7dadf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c19995e1dbc4e61a317356713e7dadf 2024-12-10T00:15:29,181 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de1b03cb10a30c0ceaa7548777a2855e/C of de1b03cb10a30c0ceaa7548777a2855e into 9c19995e1dbc4e61a317356713e7dadf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:29,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:29,181 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., storeName=de1b03cb10a30c0ceaa7548777a2855e/C, priority=13, startTime=1733789728672; duration=0sec 2024-12-10T00:15:29,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:29,181 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de1b03cb10a30c0ceaa7548777a2855e:C 2024-12-10T00:15:29,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44746 deadline: 1733789789259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44728 deadline: 1733789789259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44756 deadline: 1733789789259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,261 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,261 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,262 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in de1b03cb10a30c0ceaa7548777a2855e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789789349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/b886f9839ca24f499396794f0a1fe121 2024-12-10T00:15:29,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f21d2bf33a2431181b01e7119f517c3 is 50, key is test_row_0/C:col10/1733789728657/Put/seqid=0 2024-12-10T00:15:29,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742153_1329 (size=12301) 2024-12-10T00:15:29,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:29,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44766 deadline: 1733789789859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:15:29,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f21d2bf33a2431181b01e7119f517c3 2024-12-10T00:15:29,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/0c5430ae6fed48d1a4c3fd271a1f5d8b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0c5430ae6fed48d1a4c3fd271a1f5d8b 2024-12-10T00:15:29,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0c5430ae6fed48d1a4c3fd271a1f5d8b, entries=200, sequenceid=517, filesize=14.4 K 2024-12-10T00:15:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/b886f9839ca24f499396794f0a1fe121 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/b886f9839ca24f499396794f0a1fe121 2024-12-10T00:15:29,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:29,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/b886f9839ca24f499396794f0a1fe121, entries=150, sequenceid=517, filesize=12.0 K 2024-12-10T00:15:29,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. as already flushing 2024-12-10T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:29,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:29,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/3f21d2bf33a2431181b01e7119f517c3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f21d2bf33a2431181b01e7119f517c3 2024-12-10T00:15:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:30,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f21d2bf33a2431181b01e7119f517c3, entries=150, sequenceid=517, filesize=12.0 K 2024-12-10T00:15:30,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for de1b03cb10a30c0ceaa7548777a2855e in 1331ms, sequenceid=517, compaction requested=false 2024-12-10T00:15:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-10T00:15:30,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:30,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:30,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/81a2a6d4e6df453980b08837b22ff154 is 50, key is test_row_0/A:col10/1733789728699/Put/seqid=0 2024-12-10T00:15:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742154_1330 (size=12301) 2024-12-10T00:15:30,180 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/81a2a6d4e6df453980b08837b22ff154 2024-12-10T00:15:30,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7a01feb0b3be4afc86426cde65aa7eb5 is 50, key is test_row_0/B:col10/1733789728699/Put/seqid=0 2024-12-10T00:15:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742155_1331 (size=12301) 2024-12-10T00:15:30,256 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-12-10T00:15:30,256 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,256 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,256 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:30,257 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-12-10T00:15:30,257 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,257 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,257 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:30,258 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-12-10T00:15:30,258 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,258 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,258 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,258 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-10T00:15:30,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:30,258 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,258 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,258 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:30,260 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7a01feb0b3be4afc86426cde65aa7eb5 2024-12-10T00:15:30,261 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-12-10T00:15:30,261 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,261 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,261 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/9c7a2d45d6844a0097908a4e21f35cfd is 50, key is test_row_0/C:col10/1733789728699/Put/seqid=0 2024-12-10T00:15:30,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742156_1332 (size=12301) 2024-12-10T00:15:30,287 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/9c7a2d45d6844a0097908a4e21f35cfd 2024-12-10T00:15:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/81a2a6d4e6df453980b08837b22ff154 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/81a2a6d4e6df453980b08837b22ff154 2024-12-10T00:15:30,297 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/81a2a6d4e6df453980b08837b22ff154, entries=150, sequenceid=531, filesize=12.0 K 2024-12-10T00:15:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/7a01feb0b3be4afc86426cde65aa7eb5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7a01feb0b3be4afc86426cde65aa7eb5 2024-12-10T00:15:30,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7a01feb0b3be4afc86426cde65aa7eb5, entries=150, sequenceid=531, filesize=12.0 K 2024-12-10T00:15:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/9c7a2d45d6844a0097908a4e21f35cfd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c7a2d45d6844a0097908a4e21f35cfd 2024-12-10T00:15:30,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c7a2d45d6844a0097908a4e21f35cfd, entries=150, sequenceid=531, filesize=12.0 K 2024-12-10T00:15:30,313 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for de1b03cb10a30c0ceaa7548777a2855e in 161ms, sequenceid=531, compaction requested=true 2024-12-10T00:15:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for de1b03cb10a30c0ceaa7548777a2855e: 2024-12-10T00:15:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-10T00:15:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-10T00:15:30,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-10T00:15:30,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-12-10T00:15:30,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7170 sec 2024-12-10T00:15:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T00:15:30,751 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:30,873 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-12-10T00:15:30,874 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:30,874 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,874 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:30,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,272 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-12-10T00:15:31,272 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:31,272 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,272 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,281 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-12-10T00:15:31,281 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:31,281 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,281 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,292 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-12-10T00:15:31,292 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:31,292 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,293 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,630 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-12-10T00:15:31,630 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:31,630 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,630 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,630 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3825 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3698 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3634 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3831 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3701 2024-12-10T00:15:31,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:15:31,631 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:15:31,631 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:15:31,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:31,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:31,632 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:15:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:31,635 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789731635"}]},"ts":"1733789731635"} 2024-12-10T00:15:31,637 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:15:31,637 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:15:31,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:15:31,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, UNASSIGN}] 2024-12-10T00:15:31,640 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, UNASSIGN 2024-12-10T00:15:31,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de1b03cb10a30c0ceaa7548777a2855e, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:31,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, UNASSIGN because future has completed 2024-12-10T00:15:31,642 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:15:31,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:31,794 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:31,794 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing de1b03cb10a30c0ceaa7548777a2855e, disabling compactions & flushes 2024-12-10T00:15:31,795 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. after waiting 0 ms 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:31,795 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing de1b03cb10a30c0ceaa7548777a2855e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=A 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=B 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK de1b03cb10a30c0ceaa7548777a2855e, store=C 2024-12-10T00:15:31,795 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:31,798 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d018571ed32e4f6bb751a060ed72a350 is 50, key is test_row_0/A:col10/1733789731270/Put/seqid=0 2024-12-10T00:15:31,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742157_1333 (size=12301) 2024-12-10T00:15:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:32,203 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d018571ed32e4f6bb751a060ed72a350 2024-12-10T00:15:32,209 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/2341c27bb24748fbb12751e34ee5d06b is 50, key is test_row_0/B:col10/1733789731270/Put/seqid=0 2024-12-10T00:15:32,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742158_1334 (size=12301) 2024-12-10T00:15:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:32,613 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/2341c27bb24748fbb12751e34ee5d06b 2024-12-10T00:15:32,620 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4b5dd282be3040b180ae920a549fbbec is 50, key is test_row_0/C:col10/1733789731270/Put/seqid=0 2024-12-10T00:15:32,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742159_1335 (size=12301) 2024-12-10T00:15:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:33,025 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4b5dd282be3040b180ae920a549fbbec 2024-12-10T00:15:33,029 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/A/d018571ed32e4f6bb751a060ed72a350 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d018571ed32e4f6bb751a060ed72a350 2024-12-10T00:15:33,033 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d018571ed32e4f6bb751a060ed72a350, entries=150, sequenceid=539, filesize=12.0 K 2024-12-10T00:15:33,033 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/B/2341c27bb24748fbb12751e34ee5d06b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/2341c27bb24748fbb12751e34ee5d06b 2024-12-10T00:15:33,037 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/2341c27bb24748fbb12751e34ee5d06b, entries=150, sequenceid=539, filesize=12.0 K 2024-12-10T00:15:33,038 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/.tmp/C/4b5dd282be3040b180ae920a549fbbec as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4b5dd282be3040b180ae920a549fbbec 2024-12-10T00:15:33,042 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4b5dd282be3040b180ae920a549fbbec, entries=150, sequenceid=539, filesize=12.0 K 2024-12-10T00:15:33,042 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for de1b03cb10a30c0ceaa7548777a2855e in 1247ms, sequenceid=539, compaction requested=true 2024-12-10T00:15:33,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/c751d62ddd774b3b982881a44d4727c2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/719570579eab457a85c5cccd86d72c00, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5edb3303e2904d9d9a4daab8b14e19b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5f81e62424764e04932e9b340f2ebb68, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f59f211a401a40848be39853785dd09d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/71773fc3b4c54673bd5c255c62b2247a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/9369652ddea7412cb785ecf8783310dc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975] to archive 2024-12-10T00:15:33,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:33,045 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/51faebf1352c47d2807843a13dfdfb98 2024-12-10T00:15:33,045 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f527898a7a2d460ba63ebc44c7d55d1f 2024-12-10T00:15:33,045 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/ee955a48c1e54ac397f325cf58b14e33 2024-12-10T00:15:33,045 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/719570579eab457a85c5cccd86d72c00 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/719570579eab457a85c5cccd86d72c00 2024-12-10T00:15:33,045 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5a4df617146b48d7863ee9aa305e6686 2024-12-10T00:15:33,046 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/8fc12432c6fe43e3b843c27ed8e78e1c 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/c751d62ddd774b3b982881a44d4727c2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/c751d62ddd774b3b982881a44d4727c2 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6c8f9d15c52b429184cf501605d83c99 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/be94343e6601422194c889080c90a087 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5edb3303e2904d9d9a4daab8b14e19b6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5edb3303e2904d9d9a4daab8b14e19b6 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6a83283a0d184346ad824e5e52b5b9a0 2024-12-10T00:15:33,047 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/6536dd1d03f04d5593b3d9e9abe2d1d1 2024-12-10T00:15:33,048 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0fe671ddf37f4a6f9eec3a9b7f1c0a4e 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/b525217d32b9419fb78905d8f8ea4227 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5f81e62424764e04932e9b340f2ebb68 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/5f81e62424764e04932e9b340f2ebb68 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/15a05c0616d84392993da669569a6859 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/688806ba6198467bbf014daff8318ecc 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/085dfe6fd2884327a550fe1e62007ff5 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d69a27940f9c4d86bf69d56558cabde6 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f59f211a401a40848be39853785dd09d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/f59f211a401a40848be39853785dd09d 2024-12-10T00:15:33,049 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7f7bca7329274b898d6750d48b72ebd4 2024-12-10T00:15:33,050 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0058029d0cc6466e806d030b4ee7542c 2024-12-10T00:15:33,050 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a96889b9261b4275ab0665d12c2c3ef0 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/9369652ddea7412cb785ecf8783310dc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/9369652ddea7412cb785ecf8783310dc 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/71773fc3b4c54673bd5c255c62b2247a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/71773fc3b4c54673bd5c255c62b2247a 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/58094d9d81be4067af6e82c74a114d44 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a517372f1735457f8f7274ab28e12145 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/a5e7f586251d46eb8ccf7bd9c0b529e8 2024-12-10T00:15:33,051 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0b6498c645c34340b16e00a9682e2975 2024-12-10T00:15:33,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4c135be216a84bcbbd61b09b7e4f5345, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/86c72046315142ed85bc4ad00bc08dd1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/371a3dbb15a8466fb8908cd2d42c475f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/a9c4429f4e9b4f0595cbce597160c165, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/cff43c31f6f1465fb1290dfeed2245eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/bdf62cc879d848268ed5110ff0f4000a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/37286ad7010c4400b4e22d3603210d6f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/3f1d584a455b4db3a3edae367f9fece5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7] to archive 2024-12-10T00:15:33,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:33,055 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/6fc7d714efad48c79852cd2895a94527 2024-12-10T00:15:33,055 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4c135be216a84bcbbd61b09b7e4f5345 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4c135be216a84bcbbd61b09b7e4f5345 2024-12-10T00:15:33,055 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/edda83b93a76411fa08eadf672d74b71 2024-12-10T00:15:33,056 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/ef629794d72c418aa7af254bfdbac2f0 2024-12-10T00:15:33,056 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/36bc9d9142bc474d865eec9da2bc2809 2024-12-10T00:15:33,056 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/5b7c4bd1f2e142d0ab54a68e7c1706ed 2024-12-10T00:15:33,056 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/55b3f6b75ce4417d8cdde1bbb2e19595 2024-12-10T00:15:33,057 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/86c72046315142ed85bc4ad00bc08dd1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/86c72046315142ed85bc4ad00bc08dd1 2024-12-10T00:15:33,058 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/199a307ea18f45978a94edd973c9dfee 2024-12-10T00:15:33,058 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/371a3dbb15a8466fb8908cd2d42c475f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/371a3dbb15a8466fb8908cd2d42c475f 2024-12-10T00:15:33,058 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4f871e73fcb74991b9cd67c0fe3ca08e 2024-12-10T00:15:33,059 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4d1e4ebaae4a494887ef295eb4a4b3f9 2024-12-10T00:15:33,059 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/a9c4429f4e9b4f0595cbce597160c165 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/a9c4429f4e9b4f0595cbce597160c165 2024-12-10T00:15:33,059 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/62f2a06f83204e4e98240d93a83a6bf9 2024-12-10T00:15:33,059 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/21b08f14abf44349be6b32f8229eda50 2024-12-10T00:15:33,060 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/cff43c31f6f1465fb1290dfeed2245eb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/cff43c31f6f1465fb1290dfeed2245eb 2024-12-10T00:15:33,060 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/1b2c0cbd27ff4c57894e01339a679a4d 2024-12-10T00:15:33,060 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/4ceeb07efeef41a68dd2113ae3a5a638 2024-12-10T00:15:33,060 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/bdf62cc879d848268ed5110ff0f4000a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/bdf62cc879d848268ed5110ff0f4000a 2024-12-10T00:15:33,061 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7aa07f0c0a4e432dbb0cd2106d35468e 2024-12-10T00:15:33,061 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/d2b956e18f574d23bbc45184b0b48720 2024-12-10T00:15:33,061 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/e46e5e634ab94d359b846e5a2644f2dc 2024-12-10T00:15:33,061 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/37286ad7010c4400b4e22d3603210d6f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/37286ad7010c4400b4e22d3603210d6f 2024-12-10T00:15:33,061 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7f51ae0deeb640f990cc8ff0d7cbdda4 2024-12-10T00:15:33,062 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/de3777810f544defa1b0fcc351dbc1fb 2024-12-10T00:15:33,062 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71dd05d5a9944e1ea8836da7e5b19960 2024-12-10T00:15:33,062 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/914b6d93f1c04ea89842f1c1b523965e 2024-12-10T00:15:33,062 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/3f1d584a455b4db3a3edae367f9fece5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/3f1d584a455b4db3a3edae367f9fece5 2024-12-10T00:15:33,062 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/841766aa96054fb08651b5ce9bd056c7 2024-12-10T00:15:33,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/828b795fed374a0bafa4025216769ed9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cfdb1d5c05e04014ae4537d815ac04b9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3b22586d1d3340f18768c110e74eb718, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8a40291ed88443d898cc9cd5d2039a17, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/170b18d980c2496688b541242bdfa7cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/fcfca2f168004992a54ec408ec029b5d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2b55740d7f9f454f9d2d58770136f5c6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b] to archive 2024-12-10T00:15:33,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:15:33,065 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6ee0856637ce4f5a97a10a4f5f443912 2024-12-10T00:15:33,065 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/181e8d783f9649a0ad6fc9f20c60ed4c 2024-12-10T00:15:33,065 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/828b795fed374a0bafa4025216769ed9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/828b795fed374a0bafa4025216769ed9 2024-12-10T00:15:33,066 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/92c1bf2a8a764d83831d44163c32f1ac 2024-12-10T00:15:33,066 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f15dbb807834873b5f25d38ce010527 2024-12-10T00:15:33,066 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/04066a18bebe4ed3ac23c8dd1d9e3ea5 2024-12-10T00:15:33,066 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/68724f6e75954a31baf086182d8088ec 2024-12-10T00:15:33,066 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cfdb1d5c05e04014ae4537d815ac04b9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cfdb1d5c05e04014ae4537d815ac04b9 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3b22586d1d3340f18768c110e74eb718 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3b22586d1d3340f18768c110e74eb718 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3e8610ca807148b69ae901f021b22450 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/19313dadcf2449eea091300ab91b6471 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/d1cc48dcfc054085bdcd543dc2dea81e 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/14debc7cb4c6481590d6e78a2c012663 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8551e5d1cb1147a2bf1761eec3d27b3d 2024-12-10T00:15:33,067 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/c9a466c3bea04396b9d51a66aa004265 2024-12-10T00:15:33,068 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8a40291ed88443d898cc9cd5d2039a17 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/8a40291ed88443d898cc9cd5d2039a17 2024-12-10T00:15:33,068 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/170b18d980c2496688b541242bdfa7cf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/170b18d980c2496688b541242bdfa7cf 2024-12-10T00:15:33,068 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/cb76121459ce40baacc2347dbf4a9889 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2de54e4e15cd4c91bf4484d2fcaf9192 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/52238b915be245d99e6efacd5e2d36f5 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/fcfca2f168004992a54ec408ec029b5d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/fcfca2f168004992a54ec408ec029b5d 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/6f8a815625ad4197b0cb9069ac762f09 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/37e4c8376a8c42bf8bc0f333ded4fdf2 2024-12-10T00:15:33,069 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/157a40e9a3f04c6e94d4f7b31fe2ce04 2024-12-10T00:15:33,070 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/96ad8f324d514017bfc39cbb02af753e 2024-12-10T00:15:33,070 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2b55740d7f9f454f9d2d58770136f5c6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/2b55740d7f9f454f9d2d58770136f5c6 2024-12-10T00:15:33,070 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/e1c05c4a19cf4524a2ce2af481b029df 2024-12-10T00:15:33,070 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4d203e9599e4408288710b61b30da58b 2024-12-10T00:15:33,073 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=1 2024-12-10T00:15:33,074 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. 2024-12-10T00:15:33,074 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for de1b03cb10a30c0ceaa7548777a2855e: Waiting for close lock at 1733789731795Running coprocessor pre-close hooks at 1733789731795Disabling compacts and flushes for region at 1733789731795Disabling writes for close at 1733789731795Obtaining lock to block concurrent updates at 1733789731795Preparing flush snapshotting stores in de1b03cb10a30c0ceaa7548777a2855e at 1733789731795Finished memstore snapshotting TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733789731795Flushing stores of TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e. at 1733789731795Flushing de1b03cb10a30c0ceaa7548777a2855e/A: creating writer at 1733789731795Flushing de1b03cb10a30c0ceaa7548777a2855e/A: appending metadata at 1733789731798 (+3 ms)Flushing de1b03cb10a30c0ceaa7548777a2855e/A: closing flushed file at 1733789731798Flushing de1b03cb10a30c0ceaa7548777a2855e/B: creating writer at 1733789732206 (+408 ms)Flushing de1b03cb10a30c0ceaa7548777a2855e/B: appending metadata at 1733789732209 (+3 ms)Flushing de1b03cb10a30c0ceaa7548777a2855e/B: closing flushed file at 1733789732209Flushing de1b03cb10a30c0ceaa7548777a2855e/C: creating writer at 1733789732617 (+408 ms)Flushing de1b03cb10a30c0ceaa7548777a2855e/C: appending metadata at 1733789732619 (+2 ms)Flushing de1b03cb10a30c0ceaa7548777a2855e/C: closing flushed file at 1733789732619Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c7fd277: reopening flushed file at 1733789733028 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66964fae: reopening flushed file at 1733789733033 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2abc8a70: reopening flushed file at 1733789733037 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for de1b03cb10a30c0ceaa7548777a2855e in 1247ms, sequenceid=539, compaction requested=true at 1733789733042 (+5 ms)Writing region close event to WAL at 1733789733071 (+29 ms)Running coprocessor post-close hooks at 1733789733074 (+3 ms)Closed at 1733789733074 2024-12-10T00:15:33,076 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:33,076 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=de1b03cb10a30c0ceaa7548777a2855e, regionState=CLOSED 2024-12-10T00:15:33,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:33,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-10T00:15:33,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure de1b03cb10a30c0ceaa7548777a2855e, server=a67c4886b4f7,36911,1733789645811 in 1.4360 sec 2024-12-10T00:15:33,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-10T00:15:33,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de1b03cb10a30c0ceaa7548777a2855e, UNASSIGN in 1.4410 sec 2024-12-10T00:15:33,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-10T00:15:33,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4440 sec 2024-12-10T00:15:33,084 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789733084"}]},"ts":"1733789733084"} 2024-12-10T00:15:33,085 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:15:33,085 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:15:33,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4530 sec 2024-12-10T00:15:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-10T00:15:33,781 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:33,782 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:15:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,787 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-10T00:15:33,789 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,792 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:33,794 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/recovered.edits] 2024-12-10T00:15:33,795 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:15:33,798 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0c5430ae6fed48d1a4c3fd271a1f5d8b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/0c5430ae6fed48d1a4c3fd271a1f5d8b 2024-12-10T00:15:33,798 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7aa7bf7a430c4373b9574c6afe8df38f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/7aa7bf7a430c4373b9574c6afe8df38f 2024-12-10T00:15:33,798 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d018571ed32e4f6bb751a060ed72a350 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/d018571ed32e4f6bb751a060ed72a350 2024-12-10T00:15:33,798 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/81a2a6d4e6df453980b08837b22ff154 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/A/81a2a6d4e6df453980b08837b22ff154 2024-12-10T00:15:33,801 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7a01feb0b3be4afc86426cde65aa7eb5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/7a01feb0b3be4afc86426cde65aa7eb5 2024-12-10T00:15:33,801 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/2341c27bb24748fbb12751e34ee5d06b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/2341c27bb24748fbb12751e34ee5d06b 2024-12-10T00:15:33,801 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/b886f9839ca24f499396794f0a1fe121 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/b886f9839ca24f499396794f0a1fe121 2024-12-10T00:15:33,801 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71bc8655732047e79e1dea0043b49cdb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/B/71bc8655732047e79e1dea0043b49cdb 2024-12-10T00:15:33,804 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4b5dd282be3040b180ae920a549fbbec to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/4b5dd282be3040b180ae920a549fbbec 2024-12-10T00:15:33,804 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f21d2bf33a2431181b01e7119f517c3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/3f21d2bf33a2431181b01e7119f517c3 2024-12-10T00:15:33,804 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c7a2d45d6844a0097908a4e21f35cfd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c7a2d45d6844a0097908a4e21f35cfd 2024-12-10T00:15:33,805 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c19995e1dbc4e61a317356713e7dadf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/C/9c19995e1dbc4e61a317356713e7dadf 2024-12-10T00:15:33,808 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/recovered.edits/542.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e/recovered.edits/542.seqid 2024-12-10T00:15:33,808 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/de1b03cb10a30c0ceaa7548777a2855e 2024-12-10T00:15:33,808 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:15:33,811 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,814 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:15:33,817 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:15:33,819 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,819 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:15:33,820 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789733819"}]},"ts":"9223372036854775807"} 2024-12-10T00:15:33,823 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:15:33,823 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => de1b03cb10a30c0ceaa7548777a2855e, NAME => 'TestAcidGuarantees,,1733789707999.de1b03cb10a30c0ceaa7548777a2855e.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:15:33,823 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:15:33,823 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789733823"}]},"ts":"9223372036854775807"} 2024-12-10T00:15:33,828 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:15:33,829 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-10T00:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-10T00:15:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:15:33,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:33,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=256 (was 255) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=541 (was 475) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3350 (was 2908) - AvailableMemoryMB LEAK? - 2024-12-10T00:15:33,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=256, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=541, ProcessCount=11, AvailableMemoryMB=3350 2024-12-10T00:15:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:15:33,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:15:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:33,918 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:15:33,918 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:33,919 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-10T00:15:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:33,919 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:15:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742160_1336 (size=1061) 2024-12-10T00:15:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:34,328 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:15:34,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742161_1337 (size=53) 2024-12-10T00:15:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:34,736 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:15:34,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00c63c75a41d5931b4010f9374d69cb4, disabling compactions & flushes 2024-12-10T00:15:34,737 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:34,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:34,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. after waiting 0 ms 2024-12-10T00:15:34,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:34,737 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:34,737 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00c63c75a41d5931b4010f9374d69cb4: Waiting for close lock at 1733789734736Disabling compacts and flushes for region at 1733789734736Disabling writes for close at 1733789734737 (+1 ms)Writing region close event to WAL at 1733789734737Closed at 1733789734737 2024-12-10T00:15:34,738 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:15:34,738 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789734738"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789734738"}]},"ts":"1733789734738"} 2024-12-10T00:15:34,740 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:15:34,741 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:15:34,741 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789734741"}]},"ts":"1733789734741"} 2024-12-10T00:15:34,742 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:15:34,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, ASSIGN}] 2024-12-10T00:15:34,744 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, ASSIGN 2024-12-10T00:15:34,744 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:15:34,895 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:34,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, ASSIGN because future has completed 2024-12-10T00:15:34,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:35,055 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:35,055 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:15:35,056 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,056 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:15:35,056 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,056 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,058 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,060 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:35,061 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName A 2024-12-10T00:15:35,061 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:35,062 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:35,062 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,065 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:35,065 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName B 2024-12-10T00:15:35,065 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:35,065 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:35,066 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,067 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:35,067 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName C 2024-12-10T00:15:35,067 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:35,067 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:35,068 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:35,068 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,068 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,069 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,070 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,070 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,070 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:15:35,071 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,073 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:15:35,073 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 00c63c75a41d5931b4010f9374d69cb4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62257630, jitterRate=-0.07228901982307434}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:15:35,074 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:35,074 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 00c63c75a41d5931b4010f9374d69cb4: Running coprocessor pre-open hook at 1733789735056Writing region info on filesystem at 1733789735056Initializing all the Stores at 1733789735057 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789735057Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789735058 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789735058Cleaning up temporary data from old regions at 1733789735070 (+12 ms)Running coprocessor post-open hooks at 1733789735074 (+4 ms)Region opened successfully at 1733789735074 2024-12-10T00:15:35,075 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., pid=91, masterSystemTime=1733789735050 2024-12-10T00:15:35,077 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:35,078 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:35,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:35,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:35,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-10T00:15:35,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 in 184 msec 2024-12-10T00:15:35,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-10T00:15:35,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, ASSIGN in 339 msec 2024-12-10T00:15:35,087 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:15:35,087 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789735087"}]},"ts":"1733789735087"} 2024-12-10T00:15:35,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:15:35,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:15:35,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-12-10T00:15:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-10T00:15:36,080 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:36,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:36,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:36,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:36,081 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f807ee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:36,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:36,083 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21deb5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:36,083 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:36,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:36,086 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@96387ed 2024-12-10T00:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:15:36,087 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:15:36,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:15:36,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:15:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T00:15:36,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742162_1338 (size=1097) 2024-12-10T00:15:36,501 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T00:15:36,501 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T00:15:36,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:15:36,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE}] 2024-12-10T00:15:36,509 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE 2024-12-10T00:15:36,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:36,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE because future has completed 2024-12-10T00:15:36,511 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:15:36,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:36,664 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,664 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:15:36,665 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 00c63c75a41d5931b4010f9374d69cb4, disabling compactions & flushes 2024-12-10T00:15:36,665 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:36,665 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:36,665 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. after waiting 0 ms 2024-12-10T00:15:36,665 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:36,669 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T00:15:36,670 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:36,670 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 00c63c75a41d5931b4010f9374d69cb4: Waiting for close lock at 1733789736665Running coprocessor pre-close hooks at 1733789736665Disabling compacts and flushes for region at 1733789736665Disabling writes for close at 1733789736665Writing region close event to WAL at 1733789736665Running coprocessor post-close hooks at 1733789736669 (+4 ms)Closed at 1733789736670 (+1 ms) 2024-12-10T00:15:36,670 WARN [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 00c63c75a41d5931b4010f9374d69cb4 to self. 2024-12-10T00:15:36,672 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,673 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=CLOSED 2024-12-10T00:15:36,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:36,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-10T00:15:36,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 in 164 msec 2024-12-10T00:15:36,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE; state=CLOSED, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=true 2024-12-10T00:15:36,830 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:36,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE because future has completed 2024-12-10T00:15:36,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:15:36,990 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:36,990 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:15:36,991 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,991 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:15:36,991 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,991 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,994 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,995 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:36,995 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName A 2024-12-10T00:15:36,997 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:36,997 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:36,998 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,998 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:36,998 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName B 2024-12-10T00:15:36,998 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:36,999 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:36,999 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:36,999 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:15:37,000 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00c63c75a41d5931b4010f9374d69cb4 columnFamilyName C 2024-12-10T00:15:37,000 DEBUG [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:37,000 INFO [StoreOpener-00c63c75a41d5931b4010f9374d69cb4-1 {}] regionserver.HStore(327): Store=00c63c75a41d5931b4010f9374d69cb4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:15:37,000 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,000 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,001 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,003 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,003 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,004 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:15:37,005 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,005 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 00c63c75a41d5931b4010f9374d69cb4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72344881, jitterRate=0.07802273333072662}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:15:37,005 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,006 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 00c63c75a41d5931b4010f9374d69cb4: Running coprocessor pre-open hook at 1733789736992Writing region info on filesystem at 1733789736992Initializing all the Stores at 1733789736993 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789736993Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789736993Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789736993Cleaning up temporary data from old regions at 1733789737003 (+10 ms)Running coprocessor post-open hooks at 1733789737005 (+2 ms)Region opened successfully at 1733789737006 (+1 ms) 2024-12-10T00:15:37,007 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., pid=96, masterSystemTime=1733789736984 2024-12-10T00:15:37,008 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,008 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=OPEN, openSeqNum=5, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:15:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-10T00:15:37,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 in 179 msec 2024-12-10T00:15:37,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T00:15:37,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, REOPEN/MOVE in 504 msec 2024-12-10T00:15:37,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-10T00:15:37,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-12-10T00:15:37,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-12-10T00:15:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-10T00:15:37,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:37,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517aada4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cca872e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cdc374f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e31478a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c836aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684febd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a401863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ace5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,042 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9290dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32324763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48299c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,047 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c6743e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ad3550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,050 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee5c99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22eae5d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673d55f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ba277b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266906c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c28d62a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:15:37,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:15:37,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108d7f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:15:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:37,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:15:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66bd1333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:15:37,060 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,060 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,060 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,060 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,060 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,061 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,069 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:37,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:37,070 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:37,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,072 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-10T00:15:37,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:37,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:37,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,079 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:15:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:15:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:15:37,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:15:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:15:37,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c7300a745ec84bf48a9d97a1ee706440_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789737067/Put/seqid=0 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789737076, max timestamp=1733789737097Num uniques 859; 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789737076, max timestamp=1733789737097Num uniques 859; 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789737076, max timestamp=1733789737097Num uniques 859; 2024-12-10T00:15:37,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742163_1339 (size=12154) 2024-12-10T00:15:37,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:37,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c7300a745ec84bf48a9d97a1ee706440_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c7300a745ec84bf48a9d97a1ee706440_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/17aa7acadf6d44358acda5e69692808b, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:37,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/17aa7acadf6d44358acda5e69692808b is 175, key is test_row_0/A:col10/1733789737067/Put/seqid=0 2024-12-10T00:15:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742164_1340 (size=30955) 2024-12-10T00:15:37,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/17aa7acadf6d44358acda5e69692808b 2024-12-10T00:15:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789797127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789797128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789797128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789797128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789797129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/fe0b1d950a914c88a9443d6e93e563e2 is 50, key is test_row_0/B:col10/1733789737067/Put/seqid=0 2024-12-10T00:15:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742165_1341 (size=12001) 2024-12-10T00:15:37,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/fe0b1d950a914c88a9443d6e93e563e2 2024-12-10T00:15:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:37,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a0873658daaf428ab8146d0f3fb99cce is 50, key is test_row_0/C:col10/1733789737067/Put/seqid=0 2024-12-10T00:15:37,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742166_1342 (size=12001) 2024-12-10T00:15:37,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a0873658daaf428ab8146d0f3fb99cce 2024-12-10T00:15:37,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/17aa7acadf6d44358acda5e69692808b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b 2024-12-10T00:15:37,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b, entries=150, sequenceid=14, filesize=30.2 K 2024-12-10T00:15:37,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/fe0b1d950a914c88a9443d6e93e563e2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2 2024-12-10T00:15:37,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T00:15:37,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a0873658daaf428ab8146d0f3fb99cce as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce 2024-12-10T00:15:37,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T00:15:37,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 00c63c75a41d5931b4010f9374d69cb4 in 181ms, sequenceid=14, compaction requested=false 2024-12-10T00:15:37,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:37,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789797264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789797264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789797272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210715f41ee1e714e4ca986e6419fda8a6f_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789737258/Put/seqid=0 2024-12-10T00:15:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742167_1343 (size=14594) 2024-12-10T00:15:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789797276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789797276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789797379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789797379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789797379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789797390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789797390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789797589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789797600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789797600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789797610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789797610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:37,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210715f41ee1e714e4ca986e6419fda8a6f_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210715f41ee1e714e4ca986e6419fda8a6f_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:37,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/98214ca7eab54d51a8aa172f6b49a1d7, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:37,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/98214ca7eab54d51a8aa172f6b49a1d7 is 175, key is test_row_0/A:col10/1733789737258/Put/seqid=0 2024-12-10T00:15:37,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742168_1344 (size=39549) 2024-12-10T00:15:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:37,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789797899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789797910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789797910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789797929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789797929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:37,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:37,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:37,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:37,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/98214ca7eab54d51a8aa172f6b49a1d7 2024-12-10T00:15:38,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/a3d03e18c6de452f858e0ca9dafc38e4 is 50, key is test_row_0/B:col10/1733789737258/Put/seqid=0 2024-12-10T00:15:38,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742169_1345 (size=12001) 2024-12-10T00:15:38,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:38,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789798410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789798419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789798420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789798440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:38,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789798440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:38,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/a3d03e18c6de452f858e0ca9dafc38e4 2024-12-10T00:15:38,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a904afb1f46a453da362432452d6279b is 50, key is test_row_0/C:col10/1733789737258/Put/seqid=0 2024-12-10T00:15:38,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742170_1346 (size=12001) 2024-12-10T00:15:38,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:15:38,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:38,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:38,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:38,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:38,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a904afb1f46a453da362432452d6279b 2024-12-10T00:15:38,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/98214ca7eab54d51a8aa172f6b49a1d7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7 2024-12-10T00:15:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7, entries=200, sequenceid=50, filesize=38.6 K 2024-12-10T00:15:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/a3d03e18c6de452f858e0ca9dafc38e4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4 2024-12-10T00:15:38,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:15:38,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a904afb1f46a453da362432452d6279b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b 2024-12-10T00:15:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T00:15:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 00c63c75a41d5931b4010f9374d69cb4 in 1694ms, sequenceid=50, compaction requested=false 2024-12-10T00:15:38,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:39,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T00:15:39,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:39,061 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:39,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7663c667283455fa831316654ff8d63_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789737274/Put/seqid=0 2024-12-10T00:15:39,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742171_1347 (size=12154) 2024-12-10T00:15:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:39,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789739458, max timestamp=1733789739472Num uniques 859; 2024-12-10T00:15:39,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:39,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789739458, max timestamp=1733789739472Num uniques 859; 2024-12-10T00:15:39,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733789739458, max timestamp=1733789739472Num uniques 859; 2024-12-10T00:15:39,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:39,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-10T00:15:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:39,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7663c667283455fa831316654ff8d63_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7663c667283455fa831316654ff8d63_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/08972eb65f564b32a637bfedafc8b93f, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:39,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/08972eb65f564b32a637bfedafc8b93f is 175, key is test_row_0/A:col10/1733789737274/Put/seqid=0 2024-12-10T00:15:39,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742172_1348 (size=30955) 2024-12-10T00:15:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789799524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789799524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789799525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789799541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789799541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789799649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789799649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789799649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789799659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789799659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789799859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789799859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789799859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789799870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789799870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:39,882 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/08972eb65f564b32a637bfedafc8b93f 2024-12-10T00:15:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/37f7f9a0313845698d93bda6d02afb86 is 50, key is test_row_0/B:col10/1733789737274/Put/seqid=0 2024-12-10T00:15:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742173_1349 (size=12001) 2024-12-10T00:15:39,893 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/37f7f9a0313845698d93bda6d02afb86 2024-12-10T00:15:39,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b1ed6e274c424681a110815e8a6e54a6 is 50, key is test_row_0/C:col10/1733789737274/Put/seqid=0 2024-12-10T00:15:39,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742174_1350 (size=12001) 2024-12-10T00:15:40,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789800169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789800169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789800170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789800180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789800190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,313 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b1ed6e274c424681a110815e8a6e54a6 2024-12-10T00:15:40,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/08972eb65f564b32a637bfedafc8b93f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f 2024-12-10T00:15:40,328 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f, entries=150, sequenceid=60, filesize=30.2 K 2024-12-10T00:15:40,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/37f7f9a0313845698d93bda6d02afb86 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86 2024-12-10T00:15:40,340 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86, entries=150, sequenceid=60, filesize=11.7 K 2024-12-10T00:15:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b1ed6e274c424681a110815e8a6e54a6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6 2024-12-10T00:15:40,346 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6, entries=150, sequenceid=60, filesize=11.7 K 2024-12-10T00:15:40,347 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 00c63c75a41d5931b4010f9374d69cb4 in 1286ms, sequenceid=60, compaction requested=true 2024-12-10T00:15:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-10T00:15:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-10T00:15:40,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-10T00:15:40,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2750 sec 2024-12-10T00:15:40,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.2800 sec 2024-12-10T00:15:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:40,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:40,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:40,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789800708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789800713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fa39171ec429447381e161543dfd9004_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789800713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789800719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789800722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742175_1351 (size=14594) 2024-12-10T00:15:40,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fa39171ec429447381e161543dfd9004_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fa39171ec429447381e161543dfd9004_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:40,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e9707ae1072241059c593d64ab083e70, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:40,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e9707ae1072241059c593d64ab083e70 is 175, key is test_row_0/A:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:40,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742176_1352 (size=39549) 2024-12-10T00:15:40,776 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e9707ae1072241059c593d64ab083e70 2024-12-10T00:15:40,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9632e0b5e44d40a4b200ff7f076f02e7 is 50, key is test_row_0/B:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:40,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742177_1353 (size=12001) 2024-12-10T00:15:40,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789800829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789800830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789800830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789800830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789800840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789801049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789801050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789801057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789801057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789801060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9632e0b5e44d40a4b200ff7f076f02e7 2024-12-10T00:15:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T00:15:41,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:41,239 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-10T00:15:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:41,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:41,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:41,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:41,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f6f6b5c54e26458f855471e9d835edc8 is 50, key is test_row_0/C:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:41,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742178_1354 (size=12001) 2024-12-10T00:15:41,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f6f6b5c54e26458f855471e9d835edc8 2024-12-10T00:15:41,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e9707ae1072241059c593d64ab083e70 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70 2024-12-10T00:15:41,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70, entries=200, sequenceid=97, filesize=38.6 K 2024-12-10T00:15:41,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9632e0b5e44d40a4b200ff7f076f02e7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7 2024-12-10T00:15:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T00:15:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f6f6b5c54e26458f855471e9d835edc8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8 2024-12-10T00:15:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8, entries=150, sequenceid=97, filesize=11.7 K 2024-12-10T00:15:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 00c63c75a41d5931b4010f9374d69cb4 in 663ms, sequenceid=97, compaction requested=true 2024-12-10T00:15:41,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:41,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:41,366 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,369 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,370 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:41,370 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:41,370 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=46.9 K 2024-12-10T00:15:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,373 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:41,374 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:41,374 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:41,374 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=137.7 K 2024-12-10T00:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,374 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0b1d950a914c88a9443d6e93e563e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733789737064 2024-12-10T00:15:41,374 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:41,374 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70] 2024-12-10T00:15:41,374 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d03e18c6de452f858e0ca9dafc38e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789737113 2024-12-10T00:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,375 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17aa7acadf6d44358acda5e69692808b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733789737064 2024-12-10T00:15:41,375 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f7f9a0313845698d93bda6d02afb86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789737261 2024-12-10T00:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,378 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9632e0b5e44d40a4b200ff7f076f02e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,378 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98214ca7eab54d51a8aa172f6b49a1d7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789737113 2024-12-10T00:15:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,379 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08972eb65f564b32a637bfedafc8b93f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789737261 2024-12-10T00:15:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,380 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9707ae1072241059c593d64ab083e70, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,391 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,394 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T00:15:41,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/f014e78c875748608fbfe166b36a26eb is 50, key is test_row_0/B:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:41,395 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:15:41,395 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210047beefd85b74f728038c512b91ed649_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,404 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210047beefd85b74f728038c512b91ed649_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:41,404 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210047beefd85b74f728038c512b91ed649_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742179_1355 (size=12139) 2024-12-10T00:15:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100249e1dafb734492b14c89d75c97474c_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789740718/Put/seqid=0 2024-12-10T00:15:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742180_1356 (size=4469) 2024-12-10T00:15:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,422 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/f014e78c875748608fbfe166b36a26eb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/f014e78c875748608fbfe166b36a26eb 2024-12-10T00:15:41,422 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#304 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,423 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/a3c5b3a10ceb412496b9dd2e55a4f15c is 175, key is test_row_0/A:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,429 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into f014e78c875748608fbfe166b36a26eb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:41,429 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:41,429 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=12, startTime=1733789741366; duration=0sec 2024-12-10T00:15:41,429 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:41,430 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:41,430 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,445 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:41,445 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:41,445 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:41,445 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=46.9 K 2024-12-10T00:15:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,446 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a0873658daaf428ab8146d0f3fb99cce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733789737064 2024-12-10T00:15:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,453 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a904afb1f46a453da362432452d6279b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733789737113 2024-12-10T00:15:41,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742181_1357 (size=9714) 2024-12-10T00:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,457 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ed6e274c424681a110815e8a6e54a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789737261 2024-12-10T00:15:41,457 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f6b5c54e26458f855471e9d835edc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742182_1358 (size=31093) 2024-12-10T00:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,471 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/a3c5b3a10ceb412496b9dd2e55a4f15c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c 2024-12-10T00:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,478 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into a3c5b3a10ceb412496b9dd2e55a4f15c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:41,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:41,478 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=12, startTime=1733789741366; duration=0sec 2024-12-10T00:15:41,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:41,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,495 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,496 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/9aaf0a9837d54d6a8863ddf49bad3d4d is 50, key is test_row_0/C:col10/1733789740700/Put/seqid=0 2024-12-10T00:15:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742183_1359 (size=12139) 2024-12-10T00:15:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:41,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789741524, max timestamp=1733789741561Num uniques 859; 2024-12-10T00:15:41,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789741524, max timestamp=1733789741561Num uniques 859; 2024-12-10T00:15:41,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:41,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:41,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789741524, max timestamp=1733789741561Num uniques 859; 2024-12-10T00:15:41,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T00:15:41,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789801591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789741585, max timestamp=1733789741600Num uniques 1068; 2024-12-10T00:15:41,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T00:15:41,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789741585, max timestamp=1733789741600Num uniques 1009; 2024-12-10T00:15:41,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-10T00:15:41,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733789741585, max timestamp=1733789741600Num uniques 1068; 2024-12-10T00:15:41,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-10T00:15:41,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789801640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789801640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789741624, max timestamp=1733789741650Num uniques 1077; 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789741624, max timestamp=1733789741650Num uniques 1077; 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733789741624, max timestamp=1733789741650Num uniques 1077; 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T00:15:41,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-10T00:15:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789801720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789741703, max timestamp=1733789741727Num uniques 1036; 2024-12-10T00:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:41,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789741703, max timestamp=1733789741727Num uniques 1036; 2024-12-10T00:15:41,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:41,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733789741703, max timestamp=1733789741727Num uniques 1036; 2024-12-10T00:15:41,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733789741745, max timestamp=1733789741770Num uniques 1045; 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733789741745, max timestamp=1733789741770Num uniques 1045; 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733789741745, max timestamp=1733789741770Num uniques 1045; 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T00:15:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-10T00:15:41,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789801804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789801804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789801810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789801839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:41,857 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100249e1dafb734492b14c89d75c97474c_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100249e1dafb734492b14c89d75c97474c_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/ccf36353abf14d079f50284335301c52, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/ccf36353abf14d079f50284335301c52 is 175, key is test_row_0/A:col10/1733789740718/Put/seqid=0 2024-12-10T00:15:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742184_1360 (size=22361) 2024-12-10T00:15:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:41,925 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/9aaf0a9837d54d6a8863ddf49bad3d4d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9aaf0a9837d54d6a8863ddf49bad3d4d 2024-12-10T00:15:41,929 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into 9aaf0a9837d54d6a8863ddf49bad3d4d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:41,929 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:41,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,929 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=12, startTime=1733789741366; duration=0sec 2024-12-10T00:15:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789801919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,930 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:41,930 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:41,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789801919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789801919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:41,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789801930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789802050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789802139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789802139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789802139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789802140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/ccf36353abf14d079f50284335301c52 2024-12-10T00:15:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0b0f5a78c29244129d3a6eb054ffd98d is 50, key is test_row_0/B:col10/1733789740718/Put/seqid=0 2024-12-10T00:15:42,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742185_1361 (size=9657) 2024-12-10T00:15:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789802369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789802449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789802449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789802460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789802460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,678 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0b0f5a78c29244129d3a6eb054ffd98d 2024-12-10T00:15:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/6864015a378249e09f9bc9fde888d2a8 is 50, key is test_row_0/C:col10/1733789740718/Put/seqid=0 2024-12-10T00:15:42,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742186_1362 (size=9657) 2024-12-10T00:15:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789802889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789802960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789802960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789802969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789802979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:43,092 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/6864015a378249e09f9bc9fde888d2a8 2024-12-10T00:15:43,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/ccf36353abf14d079f50284335301c52 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52 2024-12-10T00:15:43,099 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52, entries=100, sequenceid=105, filesize=21.8 K 2024-12-10T00:15:43,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0b0f5a78c29244129d3a6eb054ffd98d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d 2024-12-10T00:15:43,102 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d, entries=100, sequenceid=105, filesize=9.4 K 2024-12-10T00:15:43,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/6864015a378249e09f9bc9fde888d2a8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8 2024-12-10T00:15:43,106 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8, entries=100, sequenceid=105, filesize=9.4 K 2024-12-10T00:15:43,107 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 00c63c75a41d5931b4010f9374d69cb4 in 1712ms, sequenceid=105, compaction requested=false 2024-12-10T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-10T00:15:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-10T00:15:43,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T00:15:43,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-12-10T00:15:43,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8710 sec 2024-12-10T00:15:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T00:15:43,389 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:43,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-10T00:15:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:43,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:43,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:43,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:43,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T00:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:43,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-12-10T00:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:43,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105961d57ed48b4e29ab37f3a9554099f5_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789741802/Put/seqid=0 2024-12-10T00:15:43,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742187_1363 (size=12304) 2024-12-10T00:15:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:43,562 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105961d57ed48b4e29ab37f3a9554099f5_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105961d57ed48b4e29ab37f3a9554099f5_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/9e95fefd03d14d1691cb375f434ec141, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/9e95fefd03d14d1691cb375f434ec141 is 175, key is test_row_0/A:col10/1733789741802/Put/seqid=0 2024-12-10T00:15:43,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742188_1364 (size=31105) 2024-12-10T00:15:43,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:43,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789803937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:43,968 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/9e95fefd03d14d1691cb375f434ec141 2024-12-10T00:15:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ed57632653e34830998c0ae8d5fad0c7 is 50, key is test_row_0/B:col10/1733789741802/Put/seqid=0 2024-12-10T00:15:43,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742189_1365 (size=12151) 2024-12-10T00:15:43,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789803979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:43,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789803979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:43,989 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:43,990 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:43,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789803989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:43,996 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:44,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789804000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:44,005 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789804059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:44,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789804270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:44,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ed57632653e34830998c0ae8d5fad0c7 2024-12-10T00:15:44,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/55af62cfa9314e3a85a13130b6633464 is 50, key is test_row_0/C:col10/1733789741802/Put/seqid=0 2024-12-10T00:15:44,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742190_1366 (size=12151) 2024-12-10T00:15:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789804579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:44,794 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/55af62cfa9314e3a85a13130b6633464 2024-12-10T00:15:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/9e95fefd03d14d1691cb375f434ec141 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141 2024-12-10T00:15:44,802 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141, entries=150, sequenceid=214, filesize=30.4 K 2024-12-10T00:15:44,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ed57632653e34830998c0ae8d5fad0c7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7 2024-12-10T00:15:44,806 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7, entries=150, sequenceid=214, filesize=11.9 K 2024-12-10T00:15:44,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/55af62cfa9314e3a85a13130b6633464 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464 2024-12-10T00:15:44,811 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464, entries=150, sequenceid=214, filesize=11.9 K 2024-12-10T00:15:44,812 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=33.54 KB/34350 for 00c63c75a41d5931b4010f9374d69cb4 in 1267ms, sequenceid=214, compaction requested=true 2024-12-10T00:15:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:44,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-10T00:15:44,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-10T00:15:44,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-10T00:15:44,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-12-10T00:15:44,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4330 sec 2024-12-10T00:15:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:45,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:45,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:45,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7bc03a698674a9f989e5317c1638788_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:45,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742191_1367 (size=14794) 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789745203, max timestamp=1733789745231Num uniques 859; 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789745203, max timestamp=1733789745231Num uniques 859; 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789745203, max timestamp=1733789745231Num uniques 859; 2024-12-10T00:15:45,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789805358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789805469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:45,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:45,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7bc03a698674a9f989e5317c1638788_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7bc03a698674a9f989e5317c1638788_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:45,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/6c75e713eb434cc4a27a330d42f283fc, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:45,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/6c75e713eb434cc4a27a330d42f283fc is 175, key is test_row_0/A:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742192_1368 (size=39749) 2024-12-10T00:15:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T00:15:45,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:45,540 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-10T00:15:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:45,541 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:45,542 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:45,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:45,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:45,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789805690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:45,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:45,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:45,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/6c75e713eb434cc4a27a330d42f283fc 2024-12-10T00:15:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/5443abda3782406685dac67c965ca9f5 is 50, key is test_row_0/B:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:45,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742193_1369 (size=12151) 2024-12-10T00:15:46,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789806009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789806009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789806010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,021 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:46,021 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789806020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789806020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,032 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,033 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:46,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/5443abda3782406685dac67c965ca9f5 2024-12-10T00:15:46,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b6152cbec9f140059594e48dec5f7f84 is 50, key is test_row_0/C:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:46,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742194_1370 (size=12151) 2024-12-10T00:15:46,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789806530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:46,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:46,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b6152cbec9f140059594e48dec5f7f84 2024-12-10T00:15:46,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/6c75e713eb434cc4a27a330d42f283fc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc 2024-12-10T00:15:46,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc, entries=200, sequenceid=224, filesize=38.8 K 2024-12-10T00:15:46,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/5443abda3782406685dac67c965ca9f5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5 2024-12-10T00:15:46,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T00:15:46,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b6152cbec9f140059594e48dec5f7f84 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84 2024-12-10T00:15:46,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T00:15:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:46,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 00c63c75a41d5931b4010f9374d69cb4 in 1664ms, sequenceid=224, compaction requested=true 2024-12-10T00:15:46,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:46,767 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:46,768 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:46,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:46,770 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:46,770 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:46,770 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,770 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/f014e78c875748608fbfe166b36a26eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=45.0 K 2024-12-10T00:15:46,770 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:46,770 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:46,770 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,771 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=121.4 K 2024-12-10T00:15:46,771 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,771 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc] 2024-12-10T00:15:46,771 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f014e78c875748608fbfe166b36a26eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:46,771 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c5b3a10ceb412496b9dd2e55a4f15c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:46,771 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0f5a78c29244129d3a6eb054ffd98d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733789740709 2024-12-10T00:15:46,771 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccf36353abf14d079f50284335301c52, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733789740709 2024-12-10T00:15:46,772 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ed57632653e34830998c0ae8d5fad0c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733789741792 2024-12-10T00:15:46,772 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e95fefd03d14d1691cb375f434ec141, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733789741792 2024-12-10T00:15:46,772 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5443abda3782406685dac67c965ca9f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:46,772 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c75e713eb434cc4a27a330d42f283fc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:46,780 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:46,781 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/745c534359e24cd9b8b4bd440bfbd2d9 is 50, key is test_row_0/B:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:46,782 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:46,796 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210014cd73854df4391be80e7c58bb18f24_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:46,798 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210014cd73854df4391be80e7c58bb18f24_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:46,798 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210014cd73854df4391be80e7c58bb18f24_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:46,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742195_1371 (size=12425) 2024-12-10T00:15:46,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742196_1372 (size=4469) 2024-12-10T00:15:46,817 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/745c534359e24cd9b8b4bd440bfbd2d9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/745c534359e24cd9b8b4bd440bfbd2d9 2024-12-10T00:15:46,818 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#317 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:46,818 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/38cc71d0f18c491990a1fe33d2766bbb is 175, key is test_row_0/A:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:46,823 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into 745c534359e24cd9b8b4bd440bfbd2d9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:46,824 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742197_1373 (size=31379) 2024-12-10T00:15:46,824 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=12, startTime=1733789746768; duration=0sec 2024-12-10T00:15:46,824 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:46,824 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:46,824 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:15:46,825 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:15:46,825 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:46,825 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,825 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9aaf0a9837d54d6a8863ddf49bad3d4d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=45.0 K 2024-12-10T00:15:46,826 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aaf0a9837d54d6a8863ddf49bad3d4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733789739524 2024-12-10T00:15:46,826 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6864015a378249e09f9bc9fde888d2a8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733789740709 2024-12-10T00:15:46,826 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 55af62cfa9314e3a85a13130b6633464, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733789741792 2024-12-10T00:15:46,826 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b6152cbec9f140059594e48dec5f7f84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:46,844 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:46,845 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/60e4eb57f9d6432285073d3e75677517 is 50, key is test_row_0/C:col10/1733789743930/Put/seqid=0 2024-12-10T00:15:46,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742198_1374 (size=12425) 2024-12-10T00:15:46,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:46,920 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:46,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104c1b24f01903485a954df9d0af6973b0_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789745350/Put/seqid=0 2024-12-10T00:15:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742199_1375 (size=12404) 2024-12-10T00:15:47,228 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/38cc71d0f18c491990a1fe33d2766bbb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb 2024-12-10T00:15:47,231 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into 38cc71d0f18c491990a1fe33d2766bbb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:47,231 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:47,231 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=12, startTime=1733789746767; duration=0sec 2024-12-10T00:15:47,231 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:47,231 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:47,269 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/60e4eb57f9d6432285073d3e75677517 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/60e4eb57f9d6432285073d3e75677517 2024-12-10T00:15:47,273 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into 60e4eb57f9d6432285073d3e75677517(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:47,273 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:47,273 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=12, startTime=1733789746768; duration=0sec 2024-12-10T00:15:47,273 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:47,273 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:47,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:47,335 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104c1b24f01903485a954df9d0af6973b0_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c1b24f01903485a954df9d0af6973b0_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/817903cbd61e4044bf878e8420c0531c, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/817903cbd61e4044bf878e8420c0531c is 175, key is test_row_0/A:col10/1733789745350/Put/seqid=0 2024-12-10T00:15:47,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742200_1376 (size=31205) 2024-12-10T00:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:47,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789807583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:47,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789807689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:47,742 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/817903cbd61e4044bf878e8420c0531c 2024-12-10T00:15:47,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b6d6d22c0fdb46de8edcc37a71d68b0e is 50, key is test_row_0/B:col10/1733789745350/Put/seqid=0 2024-12-10T00:15:47,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742201_1377 (size=12251) 2024-12-10T00:15:47,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789807899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b6d6d22c0fdb46de8edcc37a71d68b0e 2024-12-10T00:15:48,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f5dff77f2e6f4198b6eafe8dcea5f80c is 50, key is test_row_0/C:col10/1733789745350/Put/seqid=0 2024-12-10T00:15:48,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742202_1378 (size=12251) 2024-12-10T00:15:48,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789808209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:48,561 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f5dff77f2e6f4198b6eafe8dcea5f80c 2024-12-10T00:15:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/817903cbd61e4044bf878e8420c0531c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c 2024-12-10T00:15:48,568 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c, entries=150, sequenceid=261, filesize=30.5 K 2024-12-10T00:15:48,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b6d6d22c0fdb46de8edcc37a71d68b0e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e 2024-12-10T00:15:48,572 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e, entries=150, sequenceid=261, filesize=12.0 K 2024-12-10T00:15:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/f5dff77f2e6f4198b6eafe8dcea5f80c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c 2024-12-10T00:15:48,576 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c, entries=150, sequenceid=261, filesize=12.0 K 2024-12-10T00:15:48,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 00c63c75a41d5931b4010f9374d69cb4 in 1657ms, sequenceid=261, compaction requested=false 2024-12-10T00:15:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:48,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:48,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-10T00:15:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-10T00:15:48,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T00:15:48,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0370 sec 2024-12-10T00:15:48,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.0400 sec 2024-12-10T00:15:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:48,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:48,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108719f87e49314397a49ef2f939771fb5_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742203_1379 (size=14994) 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789748803, max timestamp=1733789748868Num uniques 859; 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789748803, max timestamp=1733789748868Num uniques 859; 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789748803, max timestamp=1733789748868Num uniques 859; 2024-12-10T00:15:48,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:15:49,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789809020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:49,132 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:49,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108719f87e49314397a49ef2f939771fb5_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108719f87e49314397a49ef2f939771fb5_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:49,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e96d302eb01d4b95ac707d8b065feff8, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:49,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e96d302eb01d4b95ac707d8b065feff8 is 175, key is test_row_0/A:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:49,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742204_1380 (size=39949) 2024-12-10T00:15:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789809139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789809349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:49,540 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e96d302eb01d4b95ac707d8b065feff8 2024-12-10T00:15:49,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b5e60858f6464d6dabec323ea4598d35 is 50, key is test_row_0/B:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:49,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742205_1381 (size=12301) 2024-12-10T00:15:49,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789809670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T00:15:49,709 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:49,710 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-10T00:15:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T00:15:49,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:49,712 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:49,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T00:15:49,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T00:15:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:49,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:49,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b5e60858f6464d6dabec323ea4598d35 2024-12-10T00:15:49,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/79463bfcd0084d828fe5e10c8a3a9c48 is 50, key is test_row_0/C:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:49,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742206_1382 (size=12301) 2024-12-10T00:15:49,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/79463bfcd0084d828fe5e10c8a3a9c48 2024-12-10T00:15:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/e96d302eb01d4b95ac707d8b065feff8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8 2024-12-10T00:15:49,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8, entries=200, sequenceid=274, filesize=39.0 K 2024-12-10T00:15:49,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/b5e60858f6464d6dabec323ea4598d35 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35 2024-12-10T00:15:49,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T00:15:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/79463bfcd0084d828fe5e10c8a3a9c48 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48 2024-12-10T00:15:50,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48, entries=150, sequenceid=274, filesize=12.0 K 2024-12-10T00:15:50,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 00c63c75a41d5931b4010f9374d69cb4 in 1280ms, sequenceid=274, compaction requested=true 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:50,004 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:50,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:50,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:50,005 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:50,005 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:50,006 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:50,006 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:50,006 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,006 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,006 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=100.1 K 2024-12-10T00:15:50,006 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/745c534359e24cd9b8b4bd440bfbd2d9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.1 K 2024-12-10T00:15:50,006 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,006 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8] 2024-12-10T00:15:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:50,007 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38cc71d0f18c491990a1fe33d2766bbb, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:50,007 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 745c534359e24cd9b8b4bd440bfbd2d9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:50,008 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 817903cbd61e4044bf878e8420c0531c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789745312 2024-12-10T00:15:50,008 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d6d22c0fdb46de8edcc37a71d68b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789745312 2024-12-10T00:15:50,008 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e60858f6464d6dabec323ea4598d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:50,008 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96d302eb01d4b95ac707d8b065feff8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:50,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T00:15:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,017 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-10T00:15:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:15:50,029 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T00:15:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109a3208b3bc944ee990dabe6ee1702d97_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789748950/Put/seqid=0 2024-12-10T00:15:50,047 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:50,047 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210ac5bb9ec2b5e4578adb332b83dfb685d_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,047 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ad119e5796c045228bb30d905cff814c is 50, key is test_row_0/B:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:50,049 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210ac5bb9ec2b5e4578adb332b83dfb685d_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,049 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ac5bb9ec2b5e4578adb332b83dfb685d_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:50,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742207_1383 (size=12454) 2024-12-10T00:15:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:50,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742208_1384 (size=4469) 2024-12-10T00:15:50,068 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#325 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:50,069 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/8a35133d22fa465badbe36618bf0a6c1 is 175, key is test_row_0/A:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:50,073 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412109a3208b3bc944ee990dabe6ee1702d97_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109a3208b3bc944ee990dabe6ee1702d97_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742209_1385 (size=12677) 2024-12-10T00:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/185387f2083246a0856c33c39b092cb3, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/185387f2083246a0856c33c39b092cb3 is 175, key is test_row_0/A:col10/1733789748950/Put/seqid=0 2024-12-10T00:15:50,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742210_1386 (size=31631) 2024-12-10T00:15:50,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742211_1387 (size=31255) 2024-12-10T00:15:50,087 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/185387f2083246a0856c33c39b092cb3 2024-12-10T00:15:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/aebd144dc8124e5fbe2d131bd9aa59b4 is 50, key is test_row_0/B:col10/1733789748950/Put/seqid=0 2024-12-10T00:15:50,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789810084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789810086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789810088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789810089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742212_1388 (size=12301) 2024-12-10T00:15:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789810179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789810200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789810210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789810210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789810214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T00:15:50,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789810409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789810420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789810420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789810429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,480 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ad119e5796c045228bb30d905cff814c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ad119e5796c045228bb30d905cff814c 2024-12-10T00:15:50,484 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into ad119e5796c045228bb30d905cff814c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:50,485 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:50,485 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=13, startTime=1733789750004; duration=0sec 2024-12-10T00:15:50,485 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:50,485 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:50,485 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:50,487 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:50,487 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:50,487 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,487 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/60e4eb57f9d6432285073d3e75677517, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.1 K 2024-12-10T00:15:50,488 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e4eb57f9d6432285073d3e75677517, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733789743930 2024-12-10T00:15:50,488 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f5dff77f2e6f4198b6eafe8dcea5f80c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789745312 2024-12-10T00:15:50,489 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 79463bfcd0084d828fe5e10c8a3a9c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:50,490 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/8a35133d22fa465badbe36618bf0a6c1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1 2024-12-10T00:15:50,502 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into 8a35133d22fa465badbe36618bf0a6c1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:50,502 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:50,502 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=13, startTime=1733789750004; duration=0sec 2024-12-10T00:15:50,502 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:50,502 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:50,504 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/aebd144dc8124e5fbe2d131bd9aa59b4 2024-12-10T00:15:50,506 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/221a3d91f30c4983a396c1fa34eba891 is 50, key is test_row_0/C:col10/1733789747581/Put/seqid=0 2024-12-10T00:15:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742213_1389 (size=12677) 2024-12-10T00:15:50,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/9d00b8fef22a42698b5920f7d9379a3e is 50, key is test_row_0/C:col10/1733789748950/Put/seqid=0 2024-12-10T00:15:50,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742214_1390 (size=12301) 2024-12-10T00:15:50,534 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/9d00b8fef22a42698b5920f7d9379a3e 2024-12-10T00:15:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/185387f2083246a0856c33c39b092cb3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3 2024-12-10T00:15:50,543 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3, entries=150, sequenceid=309, filesize=30.5 K 2024-12-10T00:15:50,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/aebd144dc8124e5fbe2d131bd9aa59b4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4 2024-12-10T00:15:50,547 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T00:15:50,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/9d00b8fef22a42698b5920f7d9379a3e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e 2024-12-10T00:15:50,555 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T00:15:50,556 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 00c63c75a41d5931b4010f9374d69cb4 in 539ms, sequenceid=309, compaction requested=false 2024-12-10T00:15:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:50,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-10T00:15:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-10T00:15:50,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-10T00:15:50,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 847 msec 2024-12-10T00:15:50,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 853 msec 2024-12-10T00:15:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:50,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:50,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:50,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121047429b7c797f415ba4a6ac249dc88042_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:50,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742215_1391 (size=14994) 2024-12-10T00:15:50,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:50,758 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121047429b7c797f415ba4a6ac249dc88042_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121047429b7c797f415ba4a6ac249dc88042_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:50,759 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/4eb51c0ebea64424a7fc4d36608a4711, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:50,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/4eb51c0ebea64424a7fc4d36608a4711 is 175, key is test_row_0/A:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:50,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742216_1392 (size=39949) 2024-12-10T00:15:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789810773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789810774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789810784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789810788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T00:15:50,850 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:50,851 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-10T00:15:50,852 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:50,853 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:50,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789810899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789810899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789810899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789810910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:50,925 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/221a3d91f30c4983a396c1fa34eba891 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/221a3d91f30c4983a396c1fa34eba891 2024-12-10T00:15:50,928 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into 221a3d91f30c4983a396c1fa34eba891(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:50,928 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:50,928 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=13, startTime=1733789750004; duration=0sec 2024-12-10T00:15:50,928 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:50,928 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:51,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789811109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789811109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789811120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789811120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:51,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/4eb51c0ebea64424a7fc4d36608a4711 2024-12-10T00:15:51,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/e67bc4ed2bac4a03b014026e6becf162 is 50, key is test_row_0/B:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742217_1393 (size=12301) 2024-12-10T00:15:51,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789811190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,197 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:51,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789811419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789811419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789811430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789811430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:51,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/e67bc4ed2bac4a03b014026e6becf162 2024-12-10T00:15:51,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/480b6332374348538784fa680dfbb629 is 50, key is test_row_0/C:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:51,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742218_1394 (size=12301) 2024-12-10T00:15:51,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/480b6332374348538784fa680dfbb629 2024-12-10T00:15:51,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/4eb51c0ebea64424a7fc4d36608a4711 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711 2024-12-10T00:15:51,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711, entries=200, sequenceid=324, filesize=39.0 K 2024-12-10T00:15:51,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/e67bc4ed2bac4a03b014026e6becf162 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162 2024-12-10T00:15:51,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:51,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T00:15:51,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/480b6332374348538784fa680dfbb629 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629 2024-12-10T00:15:51,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T00:15:51,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 00c63c75a41d5931b4010f9374d69cb4 in 903ms, sequenceid=324, compaction requested=true 2024-12-10T00:15:51,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:51,637 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:51,638 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:51,638 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:51,638 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,638 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=100.4 K 2024-12-10T00:15:51,638 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,638 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711] 2024-12-10T00:15:51,638 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a35133d22fa465badbe36618bf0a6c1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:51,638 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185387f2083246a0856c33c39b092cb3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789748950 2024-12-10T00:15:51,639 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb51c0ebea64424a7fc4d36608a4711, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750055 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:51,640 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:51,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:51,643 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:51,643 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:51,643 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,644 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ad119e5796c045228bb30d905cff814c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.4 K 2024-12-10T00:15:51,645 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ad119e5796c045228bb30d905cff814c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:51,645 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting aebd144dc8124e5fbe2d131bd9aa59b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789748950 2024-12-10T00:15:51,645 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e67bc4ed2bac4a03b014026e6becf162, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750086 2024-12-10T00:15:51,650 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:51,657 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412104fd751f72dd24e66ae5bd1a840c14320_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:51,658 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412104fd751f72dd24e66ae5bd1a840c14320_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:51,658 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104fd751f72dd24e66ae5bd1a840c14320_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:51,658 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:51,659 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/dd638b61c0df40ada8a9e15665d8cf62 is 50, key is test_row_0/B:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742220_1396 (size=12779) 2024-12-10T00:15:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742219_1395 (size=4469) 2024-12-10T00:15:51,667 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#334 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:51,667 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/31c03d6788884231a9f576602b7a88c7 is 175, key is test_row_0/A:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:51,672 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/dd638b61c0df40ada8a9e15665d8cf62 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/dd638b61c0df40ada8a9e15665d8cf62 2024-12-10T00:15:51,679 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into dd638b61c0df40ada8a9e15665d8cf62(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:51,679 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:51,679 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=13, startTime=1733789751640; duration=0sec 2024-12-10T00:15:51,679 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:51,679 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:51,679 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:51,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742221_1397 (size=31733) 2024-12-10T00:15:51,680 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:51,680 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:51,680 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,680 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/221a3d91f30c4983a396c1fa34eba891, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.4 K 2024-12-10T00:15:51,681 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 221a3d91f30c4983a396c1fa34eba891, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733789747567 2024-12-10T00:15:51,681 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d00b8fef22a42698b5920f7d9379a3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789748950 2024-12-10T00:15:51,681 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 480b6332374348538784fa680dfbb629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750086 2024-12-10T00:15:51,689 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:51,689 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/2beaed3614db4376bb58f9bd49769972 is 50, key is test_row_0/C:col10/1733789750086/Put/seqid=0 2024-12-10T00:15:51,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742222_1398 (size=12779) 2024-12-10T00:15:51,711 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/2beaed3614db4376bb58f9bd49769972 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/2beaed3614db4376bb58f9bd49769972 2024-12-10T00:15:51,731 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into 2beaed3614db4376bb58f9bd49769972(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:51,731 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:51,731 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=13, startTime=1733789751640; duration=0sec 2024-12-10T00:15:51,731 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:51,731 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:51,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-10T00:15:51,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:51,776 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:15:51,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:51,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121091213d0d80e74234bb232181930e4eb0_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789750786/Put/seqid=0 2024-12-10T00:15:51,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742223_1399 (size=12454) 2024-12-10T00:15:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:51,809 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121091213d0d80e74234bb232181930e4eb0_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091213d0d80e74234bb232181930e4eb0_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:51,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/96a3b74a3c26458581f2cae7713ee0ed, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:51,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/96a3b74a3c26458581f2cae7713ee0ed is 175, key is test_row_0/A:col10/1733789750786/Put/seqid=0 2024-12-10T00:15:51,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742224_1400 (size=31255) 2024-12-10T00:15:51,820 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/96a3b74a3c26458581f2cae7713ee0ed 2024-12-10T00:15:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/bb808bced6f8414e8845c206778e6e03 is 50, key is test_row_0/B:col10/1733789750786/Put/seqid=0 2024-12-10T00:15:51,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742225_1401 (size=12301) 2024-12-10T00:15:51,846 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/bb808bced6f8414e8845c206778e6e03 2024-12-10T00:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 is 50, key is test_row_0/C:col10/1733789750786/Put/seqid=0 2024-12-10T00:15:51,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742226_1402 (size=12301) 2024-12-10T00:15:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:51,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789811955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789811958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789811958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789811963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:52,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789812070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789812070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789812070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789812070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,084 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/31c03d6788884231a9f576602b7a88c7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7 2024-12-10T00:15:52,087 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into 31c03d6788884231a9f576602b7a88c7(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:52,088 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:52,088 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=13, startTime=1733789751636; duration=0sec 2024-12-10T00:15:52,088 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:52,088 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:52,262 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 2024-12-10T00:15:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/96a3b74a3c26458581f2cae7713ee0ed as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed 2024-12-10T00:15:52,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed, entries=150, sequenceid=346, filesize=30.5 K 2024-12-10T00:15:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/bb808bced6f8414e8845c206778e6e03 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03 2024-12-10T00:15:52,273 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03, entries=150, sequenceid=346, filesize=12.0 K 2024-12-10T00:15:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 2024-12-10T00:15:52,276 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0, entries=150, sequenceid=346, filesize=12.0 K 2024-12-10T00:15:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T00:15:52,277 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 00c63c75a41d5931b4010f9374d69cb4 in 501ms, sequenceid=346, compaction requested=false 2024-12-10T00:15:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-10T00:15:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-10T00:15:52,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-10T00:15:52,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-12-10T00:15:52,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4300 sec 2024-12-10T00:15:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:52,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:15:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:52,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:52,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210615601412e6a47eab9333c7d922025a8_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742227_1403 (size=14994) 2024-12-10T00:15:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789812309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789812310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789812311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789812318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789812430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789812430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789812430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789812430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789812640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789812649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789812649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789812649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:52,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210615601412e6a47eab9333c7d922025a8_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210615601412e6a47eab9333c7d922025a8_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:52,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/0e1d89a647dc496d9fb4a7e6cc47eff5, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:52,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/0e1d89a647dc496d9fb4a7e6cc47eff5 is 175, key is test_row_0/A:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742228_1404 (size=39949) 2024-12-10T00:15:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789812950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789812960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789812960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789812960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T00:15:53,000 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:53,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:15:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-10T00:15:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:53,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:15:53,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:15:53,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:15:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:53,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/0e1d89a647dc496d9fb4a7e6cc47eff5 2024-12-10T00:15:53,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/87f2b283083f40a4aec43a460b912fa2 is 50, key is test_row_0/B:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:53,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742229_1405 (size=12301) 2024-12-10T00:15:53,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789813220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:53,227 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:53,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789813469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789813479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789813479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789813479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:53,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/87f2b283083f40a4aec43a460b912fa2 2024-12-10T00:15:53,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/866f0827180e40e48acfd4f2a8ce7535 is 50, key is test_row_0/C:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:53,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742230_1406 (size=12301) 2024-12-10T00:15:53,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:53,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:15:53,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/866f0827180e40e48acfd4f2a8ce7535 2024-12-10T00:15:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/0e1d89a647dc496d9fb4a7e6cc47eff5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5 2024-12-10T00:15:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5, entries=200, sequenceid=361, filesize=39.0 K 2024-12-10T00:15:53,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/87f2b283083f40a4aec43a460b912fa2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2 2024-12-10T00:15:53,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T00:15:53,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/866f0827180e40e48acfd4f2a8ce7535 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535 2024-12-10T00:15:53,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T00:15:53,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 00c63c75a41d5931b4010f9374d69cb4 in 1694ms, sequenceid=361, compaction requested=true 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:53,979 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:53,979 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:53,981 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:53,981 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:53,981 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,981 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=100.5 K 2024-12-10T00:15:53,981 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,981 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5] 2024-12-10T00:15:53,981 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:53,981 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:53,981 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:53,981 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/dd638b61c0df40ada8a9e15665d8cf62, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.5 K 2024-12-10T00:15:53,982 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c03d6788884231a9f576602b7a88c7, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750086 2024-12-10T00:15:53,982 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dd638b61c0df40ada8a9e15665d8cf62, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750086 2024-12-10T00:15:53,982 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a3b74a3c26458581f2cae7713ee0ed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733789750772 2024-12-10T00:15:53,983 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bb808bced6f8414e8845c206778e6e03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733789750772 2024-12-10T00:15:53,983 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1d89a647dc496d9fb4a7e6cc47eff5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:53,984 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f2b283083f40a4aec43a460b912fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:54,012 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:54,015 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a14dcfaf174544759cb741bfcc9253ef_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:54,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a14dcfaf174544759cb741bfcc9253ef_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:54,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a14dcfaf174544759cb741bfcc9253ef_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:54,023 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:54,023 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/508217098a9746cd949e5a22652e37bb is 50, key is test_row_0/B:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:54,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742231_1407 (size=4469) 2024-12-10T00:15:54,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742232_1408 (size=12881) 2024-12-10T00:15:54,050 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/508217098a9746cd949e5a22652e37bb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/508217098a9746cd949e5a22652e37bb 2024-12-10T00:15:54,054 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into 508217098a9746cd949e5a22652e37bb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:54,054 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:54,054 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=13, startTime=1733789753979; duration=0sec 2024-12-10T00:15:54,054 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:54,054 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:54,054 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:54,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:54,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:54,057 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:54,057 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/2beaed3614db4376bb58f9bd49769972, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.5 K 2024-12-10T00:15:54,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2beaed3614db4376bb58f9bd49769972, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733789750086 2024-12-10T00:15:54,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a7d5c5a995d84566b45d0aa5f5c0b5a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733789750772 2024-12-10T00:15:54,058 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 866f0827180e40e48acfd4f2a8ce7535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:54,067 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:54,067 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b5d53b85f2e04fca8e509ac9e6de1545 is 50, key is test_row_0/C:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:54,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:54,070 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121088d6d47fe6f04675b62e11a427d6ce08_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789752314/Put/seqid=0 2024-12-10T00:15:54,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742233_1409 (size=12881) 2024-12-10T00:15:54,092 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/b5d53b85f2e04fca8e509ac9e6de1545 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b5d53b85f2e04fca8e509ac9e6de1545 2024-12-10T00:15:54,115 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into b5d53b85f2e04fca8e509ac9e6de1545(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:54,115 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:54,115 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=13, startTime=1733789753979; duration=0sec 2024-12-10T00:15:54,115 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:54,115 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:54,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742234_1410 (size=12454) 2024-12-10T00:15:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:54,441 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:54,442 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/d21f1cf10a0e48b28c61a59608735b9a is 175, key is test_row_0/A:col10/1733789751957/Put/seqid=0 2024-12-10T00:15:54,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742235_1411 (size=31835) 2024-12-10T00:15:54,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. as already flushing 2024-12-10T00:15:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789814509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789814510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:54,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789814510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,522 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121088d6d47fe6f04675b62e11a427d6ce08_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121088d6d47fe6f04675b62e11a427d6ce08_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/f9fc49050fe545f6b1135a92de59c06a, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/f9fc49050fe545f6b1135a92de59c06a is 175, key is test_row_0/A:col10/1733789752314/Put/seqid=0 2024-12-10T00:15:54,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742236_1412 (size=31255) 2024-12-10T00:15:54,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789814518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789814629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789814630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789814630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789814630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789814840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789814840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789814840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789814840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:54,853 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/d21f1cf10a0e48b28c61a59608735b9a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a 2024-12-10T00:15:54,858 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into d21f1cf10a0e48b28c61a59608735b9a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:54,858 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:54,858 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=13, startTime=1733789753979; duration=0sec 2024-12-10T00:15:54,858 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:54,858 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/f9fc49050fe545f6b1135a92de59c06a 2024-12-10T00:15:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0f1955565ce142a48857caa42d593eb4 is 50, key is test_row_0/B:col10/1733789752314/Put/seqid=0 2024-12-10T00:15:54,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742237_1413 (size=12301) 2024-12-10T00:15:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789815150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789815150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789815151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789815160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,344 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0f1955565ce142a48857caa42d593eb4 2024-12-10T00:15:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/d1068af4458240c998bfef4b6186056f is 50, key is test_row_0/C:col10/1733789752314/Put/seqid=0 2024-12-10T00:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742238_1414 (size=12301) 2024-12-10T00:15:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789815659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789815659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789815670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789815680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:55,767 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/d1068af4458240c998bfef4b6186056f 2024-12-10T00:15:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/f9fc49050fe545f6b1135a92de59c06a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a 2024-12-10T00:15:55,773 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a, entries=150, sequenceid=380, filesize=30.5 K 2024-12-10T00:15:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/0f1955565ce142a48857caa42d593eb4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4 2024-12-10T00:15:55,777 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T00:15:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/d1068af4458240c998bfef4b6186056f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f 2024-12-10T00:15:55,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T00:15:55,782 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 00c63c75a41d5931b4010f9374d69cb4 in 1712ms, sequenceid=380, compaction requested=false 2024-12-10T00:15:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-10T00:15:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-10T00:15:55,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-10T00:15:55,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7810 sec 2024-12-10T00:15:55,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7850 sec 2024-12-10T00:15:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:56,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T00:15:56,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:56,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:56,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:56,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:56,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:56,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:56,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f183284dab5e4e7687b4bb7977c8f52c_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:56,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742239_1415 (size=14994) 2024-12-10T00:15:56,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f183284dab5e4e7687b4bb7977c8f52c_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f183284dab5e4e7687b4bb7977c8f52c_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:56,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/7159a933fad640f6ab532f4b5bcfc45a, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:56,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/7159a933fad640f6ab532f4b5bcfc45a is 175, key is test_row_0/A:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:56,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742240_1416 (size=39949) 2024-12-10T00:15:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789816704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789816705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789816706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789816710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789816820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789816820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789816820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:56,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789816829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789817030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789817030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789817030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789817051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,080 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-12-10T00:15:57,080 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:57,080 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:57,080 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,080 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-12-10T00:15:57,080 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:57,080 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,081 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:57,086 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-12-10T00:15:57,086 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:57,086 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,086 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:57,088 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-12-10T00:15:57,088 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:57,088 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,088 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:57,088 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-12-10T00:15:57,088 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:57,088 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,088 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:57,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:57,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/7159a933fad640f6ab532f4b5bcfc45a 2024-12-10T00:15:57,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ebffbb1d7da24076b39471b32e4165c2 is 50, key is test_row_0/B:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:57,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742241_1417 (size=12301) 2024-12-10T00:15:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-10T00:15:57,160 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:15:57,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49732 deadline: 1733789817239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,240 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 00c63c75a41d5931b4010f9374d69cb4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789817339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789817339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789817339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789817370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ebffbb1d7da24076b39471b32e4165c2 2024-12-10T00:15:57,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/89fac90bb6c64b399375766fd484fff8 is 50, key is test_row_0/C:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:57,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742242_1418 (size=12301) 2024-12-10T00:15:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49754 deadline: 1733789817850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49786 deadline: 1733789817850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49726 deadline: 1733789817851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:15:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733789817879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:15:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:15:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:15:57,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/89fac90bb6c64b399375766fd484fff8 2024-12-10T00:15:57,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/7159a933fad640f6ab532f4b5bcfc45a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a 2024-12-10T00:15:57,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a, entries=200, sequenceid=397, filesize=39.0 K 2024-12-10T00:15:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/ebffbb1d7da24076b39471b32e4165c2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2 2024-12-10T00:15:57,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2, entries=150, sequenceid=397, filesize=12.0 K 2024-12-10T00:15:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/89fac90bb6c64b399375766fd484fff8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8 2024-12-10T00:15:57,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8, entries=150, sequenceid=397, filesize=12.0 K 2024-12-10T00:15:57,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 00c63c75a41d5931b4010f9374d69cb4 in 1269ms, sequenceid=397, compaction requested=true 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:57,941 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:57,941 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00c63c75a41d5931b4010f9374d69cb4:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:15:57,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:57,942 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:57,942 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/A is initiating minor compaction (all files) 2024-12-10T00:15:57,942 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:57,942 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/A in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:57,942 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/B is initiating minor compaction (all files) 2024-12-10T00:15:57,942 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/B in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:57,942 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=100.6 K 2024-12-10T00:15:57,942 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/508217098a9746cd949e5a22652e37bb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.6 K 2024-12-10T00:15:57,942 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:57,942 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a] 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508217098a9746cd949e5a22652e37bb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d21f1cf10a0e48b28c61a59608735b9a, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1955565ce142a48857caa42d593eb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733789752308 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f9fc49050fe545f6b1135a92de59c06a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733789752308 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebffbb1d7da24076b39471b32e4165c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733789754508 2024-12-10T00:15:57,943 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7159a933fad640f6ab532f4b5bcfc45a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733789754508 2024-12-10T00:15:57,953 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:57,954 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#B#compaction#352 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:57,955 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/77000f1ff0404024a7e2bacd0b355147 is 50, key is test_row_0/B:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:57,955 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210198bf85b94ff46359825ed721a7b7944_00c63c75a41d5931b4010f9374d69cb4 store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:57,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742243_1419 (size=12983) 2024-12-10T00:15:57,973 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210198bf85b94ff46359825ed721a7b7944_00c63c75a41d5931b4010f9374d69cb4, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:57,973 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210198bf85b94ff46359825ed721a7b7944_00c63c75a41d5931b4010f9374d69cb4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:57,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742244_1420 (size=4469) 2024-12-10T00:15:58,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/77000f1ff0404024a7e2bacd0b355147 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/77000f1ff0404024a7e2bacd0b355147 2024-12-10T00:15:58,372 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/B of 00c63c75a41d5931b4010f9374d69cb4 into 77000f1ff0404024a7e2bacd0b355147(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:58,372 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:58,372 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/B, priority=13, startTime=1733789757941; duration=0sec 2024-12-10T00:15:58,372 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:15:58,372 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:B 2024-12-10T00:15:58,373 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:15:58,374 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:15:58,374 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 00c63c75a41d5931b4010f9374d69cb4/C is initiating minor compaction (all files) 2024-12-10T00:15:58,374 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00c63c75a41d5931b4010f9374d69cb4/C in TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:15:58,374 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b5d53b85f2e04fca8e509ac9e6de1545, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp, totalSize=36.6 K 2024-12-10T00:15:58,375 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d53b85f2e04fca8e509ac9e6de1545, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733789751956 2024-12-10T00:15:58,375 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1068af4458240c998bfef4b6186056f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733789752308 2024-12-10T00:15:58,375 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89fac90bb6c64b399375766fd484fff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733789754508 2024-12-10T00:15:58,378 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#A#compaction#353 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:58,378 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/412adbc8b64c43809ffb2820998e5ed2 is 175, key is test_row_0/A:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742245_1421 (size=31937) 2024-12-10T00:15:58,389 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00c63c75a41d5931b4010f9374d69cb4#C#compaction#354 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:15:58,389 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/412adbc8b64c43809ffb2820998e5ed2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/412adbc8b64c43809ffb2820998e5ed2 2024-12-10T00:15:58,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/fba5a65573d04fe3aa4f277fb396992b is 50, key is test_row_0/C:col10/1733789754509/Put/seqid=0 2024-12-10T00:15:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742246_1422 (size=12983) 2024-12-10T00:15:58,395 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/A of 00c63c75a41d5931b4010f9374d69cb4 into 412adbc8b64c43809ffb2820998e5ed2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:58,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:58,395 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/A, priority=13, startTime=1733789757941; duration=0sec 2024-12-10T00:15:58,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:58,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:A 2024-12-10T00:15:58,798 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/fba5a65573d04fe3aa4f277fb396992b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/fba5a65573d04fe3aa4f277fb396992b 2024-12-10T00:15:58,803 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00c63c75a41d5931b4010f9374d69cb4/C of 00c63c75a41d5931b4010f9374d69cb4 into fba5a65573d04fe3aa4f277fb396992b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:15:58,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:15:58,803 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., storeName=00c63c75a41d5931b4010f9374d69cb4/C, priority=13, startTime=1733789757941; duration=0sec 2024-12-10T00:15:58,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:15:58,803 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00c63c75a41d5931b4010f9374d69cb4:C 2024-12-10T00:15:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:58,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:15:58,862 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-12-10T00:15:58,862 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-12-10T00:15:58,862 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:58,862 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,862 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:58,862 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,862 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:58,862 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:15:58,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:58,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:58,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:15:58,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:58,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:15:58,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:15:58,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102d3cc7ff71d44cc9863609e289ebf12d_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789758860/Put/seqid=0 2024-12-10T00:15:58,871 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-12-10T00:15:58,871 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:58,871 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,871 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:58,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742247_1423 (size=12454) 2024-12-10T00:15:58,892 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-12-10T00:15:58,892 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:15:58,892 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,892 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:15:58,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:15:59,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:15:59,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102d3cc7ff71d44cc9863609e289ebf12d_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d3cc7ff71d44cc9863609e289ebf12d_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:15:59,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/af999991ff1a4ebdb724531ed3eb8995, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:15:59,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/af999991ff1a4ebdb724531ed3eb8995 is 175, key is test_row_0/A:col10/1733789758860/Put/seqid=0 2024-12-10T00:15:59,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742248_1424 (size=31255) 2024-12-10T00:15:59,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/af999991ff1a4ebdb724531ed3eb8995 2024-12-10T00:15:59,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9a21b347efb846dca4ecb28a039e8487 is 50, key is test_row_0/B:col10/1733789758860/Put/seqid=0 2024-12-10T00:15:59,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742249_1425 (size=12301) 2024-12-10T00:15:59,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9a21b347efb846dca4ecb28a039e8487 2024-12-10T00:15:59,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/8faaecfca6dd4ea5827d578aababbe30 is 50, key is test_row_0/C:col10/1733789758860/Put/seqid=0 2024-12-10T00:15:59,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742250_1426 (size=12301) 2024-12-10T00:16:00,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/8faaecfca6dd4ea5827d578aababbe30 2024-12-10T00:16:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/af999991ff1a4ebdb724531ed3eb8995 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/af999991ff1a4ebdb724531ed3eb8995 2024-12-10T00:16:00,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/af999991ff1a4ebdb724531ed3eb8995, entries=150, sequenceid=419, filesize=30.5 K 2024-12-10T00:16:00,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/9a21b347efb846dca4ecb28a039e8487 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9a21b347efb846dca4ecb28a039e8487 2024-12-10T00:16:00,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9a21b347efb846dca4ecb28a039e8487, entries=150, sequenceid=419, filesize=12.0 K 2024-12-10T00:16:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/8faaecfca6dd4ea5827d578aababbe30 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/8faaecfca6dd4ea5827d578aababbe30 2024-12-10T00:16:00,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/8faaecfca6dd4ea5827d578aababbe30, entries=150, sequenceid=419, filesize=12.0 K 2024-12-10T00:16:00,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 00c63c75a41d5931b4010f9374d69cb4 in 1269ms, sequenceid=419, compaction requested=false 2024-12-10T00:16:00,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00c63c75a41d5931b4010f9374d69cb4: 2024-12-10T00:16:03,795 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:16:07,310 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-12-10T00:16:07,310 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:07,310 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:07,310 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:07,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1456 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4366 rows 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1464 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4392 rows 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4356 rows 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1463 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4388 rows 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-12-10T00:16:07,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4374 rows 2024-12-10T00:16:07,311 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:16:07,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:16:07,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:07,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:07,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:07,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:16:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:07,314 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789767314"}]},"ts":"1733789767314"} 2024-12-10T00:16:07,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:16:07,315 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:16:07,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:16:07,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, UNASSIGN}] 2024-12-10T00:16:07,318 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, UNASSIGN 2024-12-10T00:16:07,318 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:07,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, UNASSIGN because future has completed 2024-12-10T00:16:07,320 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:16:07,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:07,473 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:07,473 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:16:07,473 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 00c63c75a41d5931b4010f9374d69cb4, disabling compactions & flushes 2024-12-10T00:16:07,473 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:16:07,473 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:16:07,473 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. after waiting 0 ms 2024-12-10T00:16:07,473 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:16:07,474 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 00c63c75a41d5931b4010f9374d69cb4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=A 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=B 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 00c63c75a41d5931b4010f9374d69cb4, store=C 2024-12-10T00:16:07,474 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:07,482 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bc31f5aaccee43a4986c95954162883f_00c63c75a41d5931b4010f9374d69cb4 is 50, key is test_row_0/A:col10/1733789767309/Put/seqid=0 2024-12-10T00:16:07,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742251_1427 (size=9914) 2024-12-10T00:16:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:07,887 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:07,896 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bc31f5aaccee43a4986c95954162883f_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bc31f5aaccee43a4986c95954162883f_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:07,897 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/eeda3ca2546b4010b021a54fdeddd9c7, store: [table=TestAcidGuarantees family=A region=00c63c75a41d5931b4010f9374d69cb4] 2024-12-10T00:16:07,898 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/eeda3ca2546b4010b021a54fdeddd9c7 is 175, key is test_row_0/A:col10/1733789767309/Put/seqid=0 2024-12-10T00:16:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742252_1428 (size=22561) 2024-12-10T00:16:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:08,303 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/eeda3ca2546b4010b021a54fdeddd9c7 2024-12-10T00:16:08,309 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/315c4ebf63174e1f9ce9a7fcd3650835 is 50, key is test_row_0/B:col10/1733789767309/Put/seqid=0 2024-12-10T00:16:08,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742253_1429 (size=9857) 2024-12-10T00:16:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:08,681 WARN [regionserver/a67c4886b4f7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-10T00:16:08,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a] to archive 2024-12-10T00:16:08,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:08,687 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/17aa7acadf6d44358acda5e69692808b 2024-12-10T00:16:08,687 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/98214ca7eab54d51a8aa172f6b49a1d7 2024-12-10T00:16:08,687 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e9707ae1072241059c593d64ab083e70 2024-12-10T00:16:08,687 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/08972eb65f564b32a637bfedafc8b93f 2024-12-10T00:16:08,687 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/a3c5b3a10ceb412496b9dd2e55a4f15c 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/6c75e713eb434cc4a27a330d42f283fc 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/8a35133d22fa465badbe36618bf0a6c1 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/9e95fefd03d14d1691cb375f434ec141 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/e96d302eb01d4b95ac707d8b065feff8 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/817903cbd61e4044bf878e8420c0531c 2024-12-10T00:16:08,690 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/185387f2083246a0856c33c39b092cb3 2024-12-10T00:16:08,691 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/38cc71d0f18c491990a1fe33d2766bbb 2024-12-10T00:16:08,691 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/ccf36353abf14d079f50284335301c52 2024-12-10T00:16:08,691 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/31c03d6788884231a9f576602b7a88c7 2024-12-10T00:16:08,691 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/0e1d89a647dc496d9fb4a7e6cc47eff5 2024-12-10T00:16:08,691 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/96a3b74a3c26458581f2cae7713ee0ed 2024-12-10T00:16:08,692 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/f9fc49050fe545f6b1135a92de59c06a 2024-12-10T00:16:08,692 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/d21f1cf10a0e48b28c61a59608735b9a 2024-12-10T00:16:08,692 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/4eb51c0ebea64424a7fc4d36608a4711 2024-12-10T00:16:08,692 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/7159a933fad640f6ab532f4b5bcfc45a 2024-12-10T00:16:08,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/f014e78c875748608fbfe166b36a26eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/745c534359e24cd9b8b4bd440bfbd2d9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ad119e5796c045228bb30d905cff814c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/dd638b61c0df40ada8a9e15665d8cf62, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/508217098a9746cd949e5a22652e37bb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2] to archive 2024-12-10T00:16:08,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:08,697 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/f014e78c875748608fbfe166b36a26eb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/f014e78c875748608fbfe166b36a26eb 2024-12-10T00:16:08,697 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/fe0b1d950a914c88a9443d6e93e563e2 2024-12-10T00:16:08,697 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/a3d03e18c6de452f858e0ca9dafc38e4 2024-12-10T00:16:08,697 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9632e0b5e44d40a4b200ff7f076f02e7 2024-12-10T00:16:08,698 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/37f7f9a0313845698d93bda6d02afb86 2024-12-10T00:16:08,698 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0b0f5a78c29244129d3a6eb054ffd98d 2024-12-10T00:16:08,698 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ed57632653e34830998c0ae8d5fad0c7 2024-12-10T00:16:08,698 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/745c534359e24cd9b8b4bd440bfbd2d9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/745c534359e24cd9b8b4bd440bfbd2d9 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b6d6d22c0fdb46de8edcc37a71d68b0e 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/aebd144dc8124e5fbe2d131bd9aa59b4 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/5443abda3782406685dac67c965ca9f5 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ad119e5796c045228bb30d905cff814c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ad119e5796c045228bb30d905cff814c 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/e67bc4ed2bac4a03b014026e6becf162 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/dd638b61c0df40ada8a9e15665d8cf62 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/dd638b61c0df40ada8a9e15665d8cf62 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/bb808bced6f8414e8845c206778e6e03 2024-12-10T00:16:08,700 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/b5e60858f6464d6dabec323ea4598d35 2024-12-10T00:16:08,701 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/ebffbb1d7da24076b39471b32e4165c2 2024-12-10T00:16:08,701 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/508217098a9746cd949e5a22652e37bb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/508217098a9746cd949e5a22652e37bb 2024-12-10T00:16:08,701 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/87f2b283083f40a4aec43a460b912fa2 2024-12-10T00:16:08,701 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/0f1955565ce142a48857caa42d593eb4 2024-12-10T00:16:08,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9aaf0a9837d54d6a8863ddf49bad3d4d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/60e4eb57f9d6432285073d3e75677517, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/221a3d91f30c4983a396c1fa34eba891, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/2beaed3614db4376bb58f9bd49769972, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b5d53b85f2e04fca8e509ac9e6de1545, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8] to archive 2024-12-10T00:16:08,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a67c4886b4f7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9aaf0a9837d54d6a8863ddf49bad3d4d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9aaf0a9837d54d6a8863ddf49bad3d4d 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a0873658daaf428ab8146d0f3fb99cce 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/6864015a378249e09f9bc9fde888d2a8 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b1ed6e274c424681a110815e8a6e54a6 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f6f6b5c54e26458f855471e9d835edc8 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/60e4eb57f9d6432285073d3e75677517 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/60e4eb57f9d6432285073d3e75677517 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a904afb1f46a453da362432452d6279b 2024-12-10T00:16:08,706 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/55af62cfa9314e3a85a13130b6633464 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b6152cbec9f140059594e48dec5f7f84 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/221a3d91f30c4983a396c1fa34eba891 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/221a3d91f30c4983a396c1fa34eba891 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/f5dff77f2e6f4198b6eafe8dcea5f80c 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/79463bfcd0084d828fe5e10c8a3a9c48 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/2beaed3614db4376bb58f9bd49769972 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/2beaed3614db4376bb58f9bd49769972 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/480b6332374348538784fa680dfbb629 2024-12-10T00:16:08,707 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/9d00b8fef22a42698b5920f7d9379a3e 2024-12-10T00:16:08,708 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/a7d5c5a995d84566b45d0aa5f5c0b5a0 2024-12-10T00:16:08,708 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b5d53b85f2e04fca8e509ac9e6de1545 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/b5d53b85f2e04fca8e509ac9e6de1545 2024-12-10T00:16:08,708 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/866f0827180e40e48acfd4f2a8ce7535 2024-12-10T00:16:08,708 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/d1068af4458240c998bfef4b6186056f 2024-12-10T00:16:08,708 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/89fac90bb6c64b399375766fd484fff8 2024-12-10T00:16:08,722 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/315c4ebf63174e1f9ce9a7fcd3650835 2024-12-10T00:16:08,728 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/5b679ad0e737441fb108bf345a99ceb5 is 50, key is test_row_0/C:col10/1733789767309/Put/seqid=0 2024-12-10T00:16:08,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742254_1430 (size=9857) 2024-12-10T00:16:09,132 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/5b679ad0e737441fb108bf345a99ceb5 2024-12-10T00:16:09,140 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/A/eeda3ca2546b4010b021a54fdeddd9c7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/eeda3ca2546b4010b021a54fdeddd9c7 2024-12-10T00:16:09,144 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/eeda3ca2546b4010b021a54fdeddd9c7, entries=100, sequenceid=425, filesize=22.0 K 2024-12-10T00:16:09,145 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/B/315c4ebf63174e1f9ce9a7fcd3650835 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/315c4ebf63174e1f9ce9a7fcd3650835 2024-12-10T00:16:09,148 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/315c4ebf63174e1f9ce9a7fcd3650835, entries=100, sequenceid=425, filesize=9.6 K 2024-12-10T00:16:09,148 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/.tmp/C/5b679ad0e737441fb108bf345a99ceb5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/5b679ad0e737441fb108bf345a99ceb5 2024-12-10T00:16:09,151 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/5b679ad0e737441fb108bf345a99ceb5, entries=100, sequenceid=425, filesize=9.6 K 2024-12-10T00:16:09,151 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00c63c75a41d5931b4010f9374d69cb4 in 1678ms, sequenceid=425, compaction requested=true 2024-12-10T00:16:09,155 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits/428.seqid, newMaxSeqId=428, maxSeqId=4 2024-12-10T00:16:09,155 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. 2024-12-10T00:16:09,155 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 00c63c75a41d5931b4010f9374d69cb4: Waiting for close lock at 1733789767473Running coprocessor pre-close hooks at 1733789767473Disabling compacts and flushes for region at 1733789767473Disabling writes for close at 1733789767473Obtaining lock to block concurrent updates at 1733789767474 (+1 ms)Preparing flush snapshotting stores in 00c63c75a41d5931b4010f9374d69cb4 at 1733789767474Finished memstore snapshotting TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733789767474Flushing stores of TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4. at 1733789767475 (+1 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/A: creating writer at 1733789767475Flushing mob file 00c63c75a41d5931b4010f9374d69cb4/A: appending metadata at 1733789767482 (+7 ms)Flushing mob file 00c63c75a41d5931b4010f9374d69cb4/A: closing flushed file at 1733789767482Flushing 00c63c75a41d5931b4010f9374d69cb4/A: appending metadata at 1733789767897 (+415 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/A: closing flushed file at 1733789767897Flushing 00c63c75a41d5931b4010f9374d69cb4/B: creating writer at 1733789768306 (+409 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/B: appending metadata at 1733789768309 (+3 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/B: closing flushed file at 1733789768309Flushing 00c63c75a41d5931b4010f9374d69cb4/C: creating writer at 1733789768725 (+416 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/C: appending metadata at 1733789768727 (+2 ms)Flushing 00c63c75a41d5931b4010f9374d69cb4/C: closing flushed file at 1733789768727Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66df9abd: reopening flushed file at 1733789769139 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d62eed: reopening flushed file at 1733789769144 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dca15b8: reopening flushed file at 1733789769148 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 00c63c75a41d5931b4010f9374d69cb4 in 1678ms, sequenceid=425, compaction requested=true at 1733789769151 (+3 ms)Writing region close event to WAL at 1733789769152 (+1 ms)Running coprocessor post-close hooks at 1733789769155 (+3 ms)Closed at 1733789769155 2024-12-10T00:16:09,157 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,158 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=00c63c75a41d5931b4010f9374d69cb4, regionState=CLOSED 2024-12-10T00:16:09,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:09,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-10T00:16:09,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 00c63c75a41d5931b4010f9374d69cb4, server=a67c4886b4f7,36911,1733789645811 in 1.8390 sec 2024-12-10T00:16:09,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-10T00:16:09,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00c63c75a41d5931b4010f9374d69cb4, UNASSIGN in 1.8440 sec 2024-12-10T00:16:09,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-10T00:16:09,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8470 sec 2024-12-10T00:16:09,165 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789769165"}]},"ts":"1733789769165"} 2024-12-10T00:16:09,166 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:16:09,166 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:16:09,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8550 sec 2024-12-10T00:16:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T00:16:09,460 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:09,460 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:16:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,461 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T00:16:09,462 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,463 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,465 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits] 2024-12-10T00:16:09,467 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/eeda3ca2546b4010b021a54fdeddd9c7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/eeda3ca2546b4010b021a54fdeddd9c7 2024-12-10T00:16:09,467 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/412adbc8b64c43809ffb2820998e5ed2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/412adbc8b64c43809ffb2820998e5ed2 2024-12-10T00:16:09,467 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/af999991ff1a4ebdb724531ed3eb8995 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/A/af999991ff1a4ebdb724531ed3eb8995 2024-12-10T00:16:09,470 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/315c4ebf63174e1f9ce9a7fcd3650835 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/315c4ebf63174e1f9ce9a7fcd3650835 2024-12-10T00:16:09,470 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9a21b347efb846dca4ecb28a039e8487 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/9a21b347efb846dca4ecb28a039e8487 2024-12-10T00:16:09,470 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/77000f1ff0404024a7e2bacd0b355147 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/B/77000f1ff0404024a7e2bacd0b355147 2024-12-10T00:16:09,473 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/5b679ad0e737441fb108bf345a99ceb5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/5b679ad0e737441fb108bf345a99ceb5 2024-12-10T00:16:09,473 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/8faaecfca6dd4ea5827d578aababbe30 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/8faaecfca6dd4ea5827d578aababbe30 2024-12-10T00:16:09,473 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/fba5a65573d04fe3aa4f277fb396992b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/C/fba5a65573d04fe3aa4f277fb396992b 2024-12-10T00:16:09,476 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits/428.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4/recovered.edits/428.seqid 2024-12-10T00:16:09,477 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,477 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:16:09,477 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:16:09,478 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T00:16:09,482 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121047429b7c797f415ba4a6ac249dc88042_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121047429b7c797f415ba4a6ac249dc88042_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c1b24f01903485a954df9d0af6973b0_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104c1b24f01903485a954df9d0af6973b0_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210715f41ee1e714e4ca986e6419fda8a6f_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210715f41ee1e714e4ca986e6419fda8a6f_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210615601412e6a47eab9333c7d922025a8_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210615601412e6a47eab9333c7d922025a8_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105961d57ed48b4e29ab37f3a9554099f5_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105961d57ed48b4e29ab37f3a9554099f5_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108719f87e49314397a49ef2f939771fb5_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108719f87e49314397a49ef2f939771fb5_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,483 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d3cc7ff71d44cc9863609e289ebf12d_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102d3cc7ff71d44cc9863609e289ebf12d_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,484 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121088d6d47fe6f04675b62e11a427d6ce08_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121088d6d47fe6f04675b62e11a427d6ce08_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,484 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109a3208b3bc944ee990dabe6ee1702d97_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412109a3208b3bc944ee990dabe6ee1702d97_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,484 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091213d0d80e74234bb232181930e4eb0_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091213d0d80e74234bb232181930e4eb0_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,484 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7663c667283455fa831316654ff8d63_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7663c667283455fa831316654ff8d63_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,484 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7bc03a698674a9f989e5317c1638788_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7bc03a698674a9f989e5317c1638788_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,485 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bc31f5aaccee43a4986c95954162883f_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210bc31f5aaccee43a4986c95954162883f_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,485 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c7300a745ec84bf48a9d97a1ee706440_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c7300a745ec84bf48a9d97a1ee706440_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,485 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f183284dab5e4e7687b4bb7977c8f52c_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f183284dab5e4e7687b4bb7977c8f52c_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,485 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fa39171ec429447381e161543dfd9004_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fa39171ec429447381e161543dfd9004_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,486 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100249e1dafb734492b14c89d75c97474c_00c63c75a41d5931b4010f9374d69cb4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100249e1dafb734492b14c89d75c97474c_00c63c75a41d5931b4010f9374d69cb4 2024-12-10T00:16:09,486 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:16:09,488 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,489 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:16:09,491 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:16:09,492 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,492 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:16:09,493 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789769492"}]},"ts":"9223372036854775807"} 2024-12-10T00:16:09,494 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:16:09,494 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00c63c75a41d5931b4010f9374d69cb4, NAME => 'TestAcidGuarantees,,1733789733916.00c63c75a41d5931b4010f9374d69cb4.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:16:09,495 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:16:09,495 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789769495"}]},"ts":"9223372036854775807"} 2024-12-10T00:16:09,496 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:16:09,497 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-12-10T00:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T00:16:09,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:16:09,570 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:09,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=255 (was 256), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 541), ProcessCount=11 (was 11), AvailableMemoryMB=2915 (was 3350) 2024-12-10T00:16:09,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=2915 2024-12-10T00:16:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:16:09,593 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:09,595 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:16:09,595 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:09,595 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-10T00:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:09,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:16:09,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742255_1431 (size=1061) 2024-12-10T00:16:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:10,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:16:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742256_1432 (size=53) 2024-12-10T00:16:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:10,411 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:16:10,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee7312073a8c1deb0dc9c8e31f81137a, disabling compactions & flushes 2024-12-10T00:16:10,412 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. after waiting 0 ms 2024-12-10T00:16:10,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,412 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee7312073a8c1deb0dc9c8e31f81137a: Waiting for close lock at 1733789770411Disabling compacts and flushes for region at 1733789770411Disabling writes for close at 1733789770412 (+1 ms)Writing region close event to WAL at 1733789770412Closed at 1733789770412 2024-12-10T00:16:10,414 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:16:10,414 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789770414"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789770414"}]},"ts":"1733789770414"} 2024-12-10T00:16:10,416 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:16:10,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:16:10,417 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789770417"}]},"ts":"1733789770417"} 2024-12-10T00:16:10,418 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:16:10,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, ASSIGN}] 2024-12-10T00:16:10,420 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, ASSIGN 2024-12-10T00:16:10,421 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:16:10,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ee7312073a8c1deb0dc9c8e31f81137a, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:10,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, ASSIGN because future has completed 2024-12-10T00:16:10,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:10,733 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,733 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:16:10,734 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,734 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:16:10,734 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,734 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,736 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,737 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:10,737 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee7312073a8c1deb0dc9c8e31f81137a columnFamilyName A 2024-12-10T00:16:10,737 DEBUG [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:10,738 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(327): Store=ee7312073a8c1deb0dc9c8e31f81137a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:10,738 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,740 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:10,740 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee7312073a8c1deb0dc9c8e31f81137a columnFamilyName B 2024-12-10T00:16:10,740 DEBUG [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:10,741 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(327): Store=ee7312073a8c1deb0dc9c8e31f81137a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:10,741 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,742 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:10,743 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee7312073a8c1deb0dc9c8e31f81137a columnFamilyName C 2024-12-10T00:16:10,743 DEBUG [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:10,744 INFO [StoreOpener-ee7312073a8c1deb0dc9c8e31f81137a-1 {}] regionserver.HStore(327): Store=ee7312073a8c1deb0dc9c8e31f81137a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:10,744 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,744 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,745 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,746 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,747 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,747 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,747 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:16:10,749 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,751 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:16:10,751 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened ee7312073a8c1deb0dc9c8e31f81137a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64093259, jitterRate=-0.04493601620197296}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:16:10,751 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:10,752 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for ee7312073a8c1deb0dc9c8e31f81137a: Running coprocessor pre-open hook at 1733789770734Writing region info on filesystem at 1733789770734Initializing all the Stores at 1733789770735 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789770735Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789770735Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789770735Cleaning up temporary data from old regions at 1733789770747 (+12 ms)Running coprocessor post-open hooks at 1733789770751 (+4 ms)Region opened successfully at 1733789770752 (+1 ms) 2024-12-10T00:16:10,753 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., pid=118, masterSystemTime=1733789770730 2024-12-10T00:16:10,755 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,755 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:10,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ee7312073a8c1deb0dc9c8e31f81137a, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:10,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:10,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-10T00:16:10,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 in 182 msec 2024-12-10T00:16:10,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-10T00:16:10,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, ASSIGN in 341 msec 2024-12-10T00:16:10,764 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:16:10,765 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789770764"}]},"ts":"1733789770764"} 2024-12-10T00:16:10,767 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:16:10,768 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:16:10,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-10T00:16:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-10T00:16:11,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:11,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491fd79c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f67ecb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,746 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:16:11,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45bb3b03 2024-12-10T00:16:11,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:16:11,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:16:11,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4faf19a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ba3d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c81c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0e0caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,754 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14dd0d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,755 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297206d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,757 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1704b10e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,758 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ca99b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ea486c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc85656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a4b7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e8629a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f3221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da1ea70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d2c903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6877c13a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed3315f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,770 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc182e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d4a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:11,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:11,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7523d22f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:11,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130b2c44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:11,774 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,774 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,774 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,774 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,774 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,775 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:11,775 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,775 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,775 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,775 DEBUG [Thread-1944 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,776 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,776 DEBUG [Thread-1944 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-10T00:16:11,776 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,776 DEBUG [Thread-1955 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,779 DEBUG [Thread-1955 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,784 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:11,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:11,785 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:11,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2] 2024-12-10T00:16:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:11,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:11,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:11,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d1b5f498a96d42f68071ebccfe84a01c is 50, key is test_row_0/A:col10/1733789771784/Put/seqid=0 2024-12-10T00:16:11,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789771811, max timestamp=1733789771839Num uniques 859; 2024-12-10T00:16:11,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:11,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789771811, max timestamp=1733789771839Num uniques 859; 2024-12-10T00:16:11,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789771811, max timestamp=1733789771839Num uniques 859; 2024-12-10T00:16:11,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:11,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742257_1433 (size=14341) 2024-12-10T00:16:11,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789831864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789831866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789831867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789831869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789831870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:11,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:11,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:11,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789831979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789831990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789831990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789831990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789831990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789832190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789832200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789832200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789832200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789832201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d1b5f498a96d42f68071ebccfe84a01c 2024-12-10T00:16:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/bdb28ca1f78a45c18a5cfac225430714 is 50, key is test_row_0/B:col10/1733789771784/Put/seqid=0 2024-12-10T00:16:12,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742258_1434 (size=12001) 2024-12-10T00:16:12,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789832509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789832509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789832509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789832509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789832519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:12,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/bdb28ca1f78a45c18a5cfac225430714 2024-12-10T00:16:12,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ab25101d667b45b6b32eabde7e10814f is 50, key is test_row_0/C:col10/1733789771784/Put/seqid=0 2024-12-10T00:16:12,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742259_1435 (size=12001) 2024-12-10T00:16:12,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:12,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:13,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:13,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789833019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:13,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789833029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:13,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789833029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789833030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:13,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789833040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:13,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ab25101d667b45b6b32eabde7e10814f 2024-12-10T00:16:13,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d1b5f498a96d42f68071ebccfe84a01c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c 2024-12-10T00:16:13,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c, entries=200, sequenceid=12, filesize=14.0 K 2024-12-10T00:16:13,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/bdb28ca1f78a45c18a5cfac225430714 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714 2024-12-10T00:16:13,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T00:16:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ab25101d667b45b6b32eabde7e10814f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f 2024-12-10T00:16:13,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T00:16:13,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ee7312073a8c1deb0dc9c8e31f81137a in 1333ms, sequenceid=12, compaction requested=false 2024-12-10T00:16:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:13,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-10T00:16:13,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:13,169 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:13,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:13,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/be15651f834d42bc98cfc302670d03f4 is 50, key is test_row_0/A:col10/1733789771855/Put/seqid=0 2024-12-10T00:16:13,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742260_1436 (size=12001) 2024-12-10T00:16:13,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/be15651f834d42bc98cfc302670d03f4 2024-12-10T00:16:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b409edb35ce844d0a9b99994b260b7d6 is 50, key is test_row_0/B:col10/1733789771855/Put/seqid=0 2024-12-10T00:16:13,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742261_1437 (size=12001) 2024-12-10T00:16:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:13,988 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b409edb35ce844d0a9b99994b260b7d6 2024-12-10T00:16:14,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a15ea28a9d2c4074af3f3fc4eeebc339 is 50, key is test_row_0/C:col10/1733789771855/Put/seqid=0 2024-12-10T00:16:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742262_1438 (size=12001) 2024-12-10T00:16:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:14,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789834061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789834063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789834063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789834080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789834081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789834190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789834190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789834190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789834200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789834200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789834400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789834400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789834400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,414 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a15ea28a9d2c4074af3f3fc4eeebc339 2024-12-10T00:16:14,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789834409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789834409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/be15651f834d42bc98cfc302670d03f4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4 2024-12-10T00:16:14,422 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:16:14,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b409edb35ce844d0a9b99994b260b7d6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6 2024-12-10T00:16:14,425 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:16:14,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a15ea28a9d2c4074af3f3fc4eeebc339 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339 2024-12-10T00:16:14,429 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T00:16:14,430 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for ee7312073a8c1deb0dc9c8e31f81137a in 1261ms, sequenceid=47, compaction requested=false 2024-12-10T00:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-10T00:16:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-10T00:16:14,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-10T00:16:14,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-12-10T00:16:14,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.6580 sec 2024-12-10T00:16:14,676 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:16:14,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:14,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:14,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 is 50, key is test_row_0/A:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:14,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742263_1439 (size=14341) 2024-12-10T00:16:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789834751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789834753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789834753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789834755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789834755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789834859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789834869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789834870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789834870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789834870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789835069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789835080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789835080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789835080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789835080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 2024-12-10T00:16:15,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/834d194e6aa843488a3aed17498f8452 is 50, key is test_row_0/B:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:15,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742264_1440 (size=12001) 2024-12-10T00:16:15,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789835379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789835390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789835390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789835390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789835390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/834d194e6aa843488a3aed17498f8452 2024-12-10T00:16:15,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/fd4752a56243462d9208ca14d9157e28 is 50, key is test_row_0/C:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:15,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742265_1441 (size=12001) 2024-12-10T00:16:15,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789835889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789835900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789835900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789835900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789835900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T00:16:15,940 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:15,940 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-10T00:16:15,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T00:16:15,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:15,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:15,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/fd4752a56243462d9208ca14d9157e28 2024-12-10T00:16:15,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 2024-12-10T00:16:15,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8, entries=200, sequenceid=60, filesize=14.0 K 2024-12-10T00:16:15,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/834d194e6aa843488a3aed17498f8452 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452 2024-12-10T00:16:15,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452, entries=150, sequenceid=60, filesize=11.7 K 2024-12-10T00:16:15,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/fd4752a56243462d9208ca14d9157e28 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28 2024-12-10T00:16:15,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28, entries=150, sequenceid=60, filesize=11.7 K 2024-12-10T00:16:15,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ee7312073a8c1deb0dc9c8e31f81137a in 1265ms, sequenceid=60, compaction requested=true 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:15,976 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:15,976 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:15,977 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:15,977 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:15,977 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.2 K 2024-12-10T00:16:15,977 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=39.7 K 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b5f498a96d42f68071ebccfe84a01c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789771784 2024-12-10T00:16:15,977 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb28ca1f78a45c18a5cfac225430714, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789771784 2024-12-10T00:16:15,978 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting be15651f834d42bc98cfc302670d03f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789771855 2024-12-10T00:16:15,978 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b409edb35ce844d0a9b99994b260b7d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789771855 2024-12-10T00:16:15,978 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6fd69c7cdd4f6eb23876c8bee5f7d8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774055 2024-12-10T00:16:15,978 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 834d194e6aa843488a3aed17498f8452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774074 2024-12-10T00:16:15,984 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#370 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:15,984 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/bd9d9f84fd7d4c7d93819ab72ca97ac5 is 50, key is test_row_0/B:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:15,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742266_1442 (size=12104) 2024-12-10T00:16:15,995 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:15,995 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/3f744356fe534771a88f79acb211b825 is 50, key is test_row_0/A:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:15,999 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/bd9d9f84fd7d4c7d93819ab72ca97ac5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bd9d9f84fd7d4c7d93819ab72ca97ac5 2024-12-10T00:16:16,004 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into bd9d9f84fd7d4c7d93819ab72ca97ac5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:16,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:16,004 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789775976; duration=0sec 2024-12-10T00:16:16,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:16,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:16,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:16,005 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:16,005 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:16,005 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:16,005 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.2 K 2024-12-10T00:16:16,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742267_1443 (size=12104) 2024-12-10T00:16:16,006 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ab25101d667b45b6b32eabde7e10814f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733789771784 2024-12-10T00:16:16,006 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a15ea28a9d2c4074af3f3fc4eeebc339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733789771855 2024-12-10T00:16:16,006 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4752a56243462d9208ca14d9157e28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774074 2024-12-10T00:16:16,011 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/3f744356fe534771a88f79acb211b825 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/3f744356fe534771a88f79acb211b825 2024-12-10T00:16:16,014 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:16,015 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/c06f70e19ab542b096ac8488a893f675 is 50, key is test_row_0/C:col10/1733789774710/Put/seqid=0 2024-12-10T00:16:16,017 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 3f744356fe534771a88f79acb211b825(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:16,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:16,017 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789775976; duration=0sec 2024-12-10T00:16:16,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:16,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742268_1444 (size=12104) 2024-12-10T00:16:16,024 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/c06f70e19ab542b096ac8488a893f675 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/c06f70e19ab542b096ac8488a893f675 2024-12-10T00:16:16,029 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into c06f70e19ab542b096ac8488a893f675(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:16,029 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:16,029 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789775976; duration=0sec 2024-12-10T00:16:16,029 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:16,029 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T00:16:16,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-10T00:16:16,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:16,094 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/64b106a1c5e747569fd749504ca84e3f is 50, key is test_row_0/A:col10/1733789774751/Put/seqid=0 2024-12-10T00:16:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742269_1445 (size=12001) 2024-12-10T00:16:16,103 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/64b106a1c5e747569fd749504ca84e3f 2024-12-10T00:16:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b956e6631b004a89a8945aed5b5ec823 is 50, key is test_row_0/B:col10/1733789774751/Put/seqid=0 2024-12-10T00:16:16,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742270_1446 (size=12001) 2024-12-10T00:16:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T00:16:16,519 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b956e6631b004a89a8945aed5b5ec823 2024-12-10T00:16:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4dbea40934734a57be89eb4791d5b89c is 50, key is test_row_0/C:col10/1733789774751/Put/seqid=0 2024-12-10T00:16:16,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742271_1447 (size=12001) 2024-12-10T00:16:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T00:16:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:16,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789836920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789836920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789836920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:16,926 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in ee7312073a8c1deb0dc9c8e31f81137a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_1' in ee7312073a8c1deb0dc9c8e31f81137a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:16,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:16,928 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4dbea40934734a57be89eb4791d5b89c 2024-12-10T00:16:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789836924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/64b106a1c5e747569fd749504ca84e3f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f 2024-12-10T00:16:16,934 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f, entries=150, sequenceid=83, filesize=11.7 K 2024-12-10T00:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/b956e6631b004a89a8945aed5b5ec823 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823 2024-12-10T00:16:16,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789836925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:16,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:16,938 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823, entries=150, sequenceid=83, filesize=11.7 K 2024-12-10T00:16:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4dbea40934734a57be89eb4791d5b89c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c 2024-12-10T00:16:16,941 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c, entries=150, sequenceid=83, filesize=11.7 K 2024-12-10T00:16:16,942 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ee7312073a8c1deb0dc9c8e31f81137a in 848ms, sequenceid=83, compaction requested=false 2024-12-10T00:16:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:16,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-10T00:16:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-10T00:16:16,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-10T00:16:16,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-12-10T00:16:16,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0070 sec 2024-12-10T00:16:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:17,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:17,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/175b951879eb4daaaaacb43435a1e06b is 50, key is test_row_0/A:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742272_1448 (size=14341) 2024-12-10T00:16:17,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/175b951879eb4daaaaacb43435a1e06b 2024-12-10T00:16:17,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/3e555fb55ef04a089ced9b2219390f7a is 50, key is test_row_0/B:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742273_1449 (size=12001) 2024-12-10T00:16:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-10T00:16:17,080 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:17,080 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-10T00:16:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:17,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:17,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:17,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:17,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789837079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789837081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789837087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:17,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789837189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789837189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789837200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T00:16:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T00:16:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789837410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789837410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789837410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/3e555fb55ef04a089ced9b2219390f7a 2024-12-10T00:16:17,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/cd9a5db5f891433fadb25aaae16ebb86 is 50, key is test_row_0/C:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742274_1450 (size=12001) 2024-12-10T00:16:17,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/cd9a5db5f891433fadb25aaae16ebb86 2024-12-10T00:16:17,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/175b951879eb4daaaaacb43435a1e06b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b 2024-12-10T00:16:17,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b, entries=200, sequenceid=96, filesize=14.0 K 2024-12-10T00:16:17,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/3e555fb55ef04a089ced9b2219390f7a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a 2024-12-10T00:16:17,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T00:16:17,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/cd9a5db5f891433fadb25aaae16ebb86 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86 2024-12-10T00:16:17,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86, entries=150, sequenceid=96, filesize=11.7 K 2024-12-10T00:16:17,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ee7312073a8c1deb0dc9c8e31f81137a in 444ms, sequenceid=96, compaction requested=true 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:17,477 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:17,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:17,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:17,478 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:17,478 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/3f744356fe534771a88f79acb211b825, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=37.5 K 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:17,478 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,478 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bd9d9f84fd7d4c7d93819ab72ca97ac5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.3 K 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f744356fe534771a88f79acb211b825, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774074 2024-12-10T00:16:17,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9d9f84fd7d4c7d93819ab72ca97ac5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774074 2024-12-10T00:16:17,479 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b956e6631b004a89a8945aed5b5ec823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733789774729 2024-12-10T00:16:17,479 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b106a1c5e747569fd749504ca84e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733789774729 2024-12-10T00:16:17,479 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 175b951879eb4daaaaacb43435a1e06b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776917 2024-12-10T00:16:17,479 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e555fb55ef04a089ced9b2219390f7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776919 2024-12-10T00:16:17,485 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:17,486 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/aba0521cbfb541ebbac170ce2ac35486 is 50, key is test_row_0/A:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,488 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:17,488 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/caaa882ead774747a9cedf5686538e94 is 50, key is test_row_0/B:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742275_1451 (size=12207) 2024-12-10T00:16:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742276_1452 (size=12207) 2024-12-10T00:16:17,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,539 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/68854c91e82948fbb48cf3e5348ef85a is 50, key is test_row_0/A:col10/1733789777063/Put/seqid=0 2024-12-10T00:16:17,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742277_1453 (size=12001) 2024-12-10T00:16:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:17,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:17,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789837745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789837746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789837751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789837870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789837870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789837870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:17,905 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/caaa882ead774747a9cedf5686538e94 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/caaa882ead774747a9cedf5686538e94 2024-12-10T00:16:17,906 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/aba0521cbfb541ebbac170ce2ac35486 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/aba0521cbfb541ebbac170ce2ac35486 2024-12-10T00:16:17,910 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into caaa882ead774747a9cedf5686538e94(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:17,910 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:17,910 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789777477; duration=0sec 2024-12-10T00:16:17,910 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:17,911 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:17,911 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:17,912 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into aba0521cbfb541ebbac170ce2ac35486(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:17,912 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:17,912 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789777477; duration=0sec 2024-12-10T00:16:17,912 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:17,912 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:17,912 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:17,912 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:17,912 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:17,912 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/c06f70e19ab542b096ac8488a893f675, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.3 K 2024-12-10T00:16:17,913 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c06f70e19ab542b096ac8488a893f675, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733789774074 2024-12-10T00:16:17,917 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dbea40934734a57be89eb4791d5b89c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733789774729 2024-12-10T00:16:17,917 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9a5db5f891433fadb25aaae16ebb86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776919 2024-12-10T00:16:17,922 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:17,922 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/8872011ee09c4726ab07d1fdf12a941c is 50, key is test_row_0/C:col10/1733789776924/Put/seqid=0 2024-12-10T00:16:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742278_1454 (size=12207) 2024-12-10T00:16:17,930 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/8872011ee09c4726ab07d1fdf12a941c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/8872011ee09c4726ab07d1fdf12a941c 2024-12-10T00:16:17,934 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 8872011ee09c4726ab07d1fdf12a941c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:17,934 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:17,934 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789777477; duration=0sec 2024-12-10T00:16:17,934 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:17,934 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:17,947 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/68854c91e82948fbb48cf3e5348ef85a 2024-12-10T00:16:17,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fca6df16d87c465580feda9fe1d51b59 is 50, key is test_row_0/B:col10/1733789777063/Put/seqid=0 2024-12-10T00:16:18,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742279_1455 (size=12001) 2024-12-10T00:16:18,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fca6df16d87c465580feda9fe1d51b59 2024-12-10T00:16:18,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0578dfc8ad1e415fb9381c01e511eac4 is 50, key is test_row_0/C:col10/1733789777063/Put/seqid=0 2024-12-10T00:16:18,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742280_1456 (size=12001) 2024-12-10T00:16:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789838090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789838089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789838096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:18,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789838399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789838399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789838409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,470 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0578dfc8ad1e415fb9381c01e511eac4 2024-12-10T00:16:18,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/68854c91e82948fbb48cf3e5348ef85a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a 2024-12-10T00:16:18,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T00:16:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fca6df16d87c465580feda9fe1d51b59 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59 2024-12-10T00:16:18,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T00:16:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0578dfc8ad1e415fb9381c01e511eac4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4 2024-12-10T00:16:18,487 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4, entries=150, sequenceid=115, filesize=11.7 K 2024-12-10T00:16:18,487 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ee7312073a8c1deb0dc9c8e31f81137a in 948ms, sequenceid=115, compaction requested=false 2024-12-10T00:16:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-10T00:16:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-10T00:16:18,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-10T00:16:18,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-12-10T00:16:18,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4110 sec 2024-12-10T00:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:18,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:16:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:18,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:18,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/128e97b578ea4e69816f36ee53ac1f0c is 50, key is test_row_0/A:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:18,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742281_1457 (size=14391) 2024-12-10T00:16:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789838941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789838948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789838949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789838952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789838975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789839060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789839060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789839060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789839060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789839090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T00:16:19,230 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:19,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-10T00:16:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:19,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:19,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:19,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789839269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789839269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789839279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789839280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789839300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/128e97b578ea4e69816f36ee53ac1f0c 2024-12-10T00:16:19,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/70a9dbf5c0ef497fac4d29adad5860ea is 50, key is test_row_0/B:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:19,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742282_1458 (size=12051) 2024-12-10T00:16:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:19,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789839589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789839589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789839590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789839590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789839609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:19,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/70a9dbf5c0ef497fac4d29adad5860ea 2024-12-10T00:16:19,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/40a368b8f52f4ad1b59b7c26fed8507f is 50, key is test_row_0/C:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:19,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742283_1459 (size=12051) 2024-12-10T00:16:19,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:19,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:19,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789840109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789840109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789840110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789840110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789840120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:20,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/40a368b8f52f4ad1b59b7c26fed8507f 2024-12-10T00:16:20,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:20,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:20,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/128e97b578ea4e69816f36ee53ac1f0c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c 2024-12-10T00:16:20,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c, entries=200, sequenceid=132, filesize=14.1 K 2024-12-10T00:16:20,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/70a9dbf5c0ef497fac4d29adad5860ea as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea 2024-12-10T00:16:20,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T00:16:20,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/40a368b8f52f4ad1b59b7c26fed8507f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f 2024-12-10T00:16:20,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T00:16:20,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ee7312073a8c1deb0dc9c8e31f81137a in 1257ms, sequenceid=132, compaction requested=true 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:20,169 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T00:16:20,170 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:20,170 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:20,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:20,171 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:20,171 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,171 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/caaa882ead774747a9cedf5686538e94, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.4 K 2024-12-10T00:16:20,171 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:20,171 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,171 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/aba0521cbfb541ebbac170ce2ac35486, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=37.7 K 2024-12-10T00:16:20,172 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting caaa882ead774747a9cedf5686538e94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776919 2024-12-10T00:16:20,172 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba0521cbfb541ebbac170ce2ac35486, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776919 2024-12-10T00:16:20,172 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fca6df16d87c465580feda9fe1d51b59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733789777063 2024-12-10T00:16:20,172 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68854c91e82948fbb48cf3e5348ef85a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733789777063 2024-12-10T00:16:20,174 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a9dbf5c0ef497fac4d29adad5860ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777745 2024-12-10T00:16:20,174 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128e97b578ea4e69816f36ee53ac1f0c, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777734 2024-12-10T00:16:20,180 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:20,180 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/677bb94f0f6245b2a2fb2e610f9462e5 is 50, key is test_row_0/A:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:20,182 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:20,183 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/47aa147d16ce436baab4108cb774294d is 50, key is test_row_0/B:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:20,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742284_1460 (size=12359) 2024-12-10T00:16:20,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742285_1461 (size=12359) 2024-12-10T00:16:20,212 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/47aa147d16ce436baab4108cb774294d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/47aa147d16ce436baab4108cb774294d 2024-12-10T00:16:20,218 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into 47aa147d16ce436baab4108cb774294d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:20,218 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:20,218 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789780169; duration=0sec 2024-12-10T00:16:20,218 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:20,218 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:20,218 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:20,220 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:20,220 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:20,220 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,220 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/8872011ee09c4726ab07d1fdf12a941c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.4 K 2024-12-10T00:16:20,220 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8872011ee09c4726ab07d1fdf12a941c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733789776919 2024-12-10T00:16:20,220 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0578dfc8ad1e415fb9381c01e511eac4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733789777063 2024-12-10T00:16:20,221 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a368b8f52f4ad1b59b7c26fed8507f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777745 2024-12-10T00:16:20,228 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:20,228 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4e2f660f09684450ac922a52a6cd0724 is 50, key is test_row_0/C:col10/1733789777745/Put/seqid=0 2024-12-10T00:16:20,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742286_1462 (size=12359) 2024-12-10T00:16:20,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-10T00:16:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,303 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:16:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:20,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:20,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:20,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:20,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ef50be2d79264f3091c738bf69121675 is 50, key is test_row_0/A:col10/1733789778950/Put/seqid=0 2024-12-10T00:16:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742287_1463 (size=12151) 2024-12-10T00:16:20,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ef50be2d79264f3091c738bf69121675 2024-12-10T00:16:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/2041307ca9054fea9bf1c027ea398869 is 50, key is test_row_0/B:col10/1733789778950/Put/seqid=0 2024-12-10T00:16:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742288_1464 (size=12151) 2024-12-10T00:16:20,392 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/2041307ca9054fea9bf1c027ea398869 2024-12-10T00:16:20,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/5b1e96d25fc44ffdb40c1ac94b8dd532 is 50, key is test_row_0/C:col10/1733789778950/Put/seqid=0 2024-12-10T00:16:20,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742289_1465 (size=12151) 2024-12-10T00:16:20,602 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/677bb94f0f6245b2a2fb2e610f9462e5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/677bb94f0f6245b2a2fb2e610f9462e5 2024-12-10T00:16:20,606 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 677bb94f0f6245b2a2fb2e610f9462e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:20,606 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:20,606 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789780169; duration=0sec 2024-12-10T00:16:20,606 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:20,606 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:20,649 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4e2f660f09684450ac922a52a6cd0724 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4e2f660f09684450ac922a52a6cd0724 2024-12-10T00:16:20,653 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 4e2f660f09684450ac922a52a6cd0724(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:20,653 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:20,653 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789780169; duration=0sec 2024-12-10T00:16:20,653 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:20,653 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:20,809 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/5b1e96d25fc44ffdb40c1ac94b8dd532 2024-12-10T00:16:20,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ef50be2d79264f3091c738bf69121675 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675 2024-12-10T00:16:20,815 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T00:16:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/2041307ca9054fea9bf1c027ea398869 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869 2024-12-10T00:16:20,819 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T00:16:20,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/5b1e96d25fc44ffdb40c1ac94b8dd532 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532 2024-12-10T00:16:20,823 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532, entries=150, sequenceid=151, filesize=11.9 K 2024-12-10T00:16:20,824 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ee7312073a8c1deb0dc9c8e31f81137a in 521ms, sequenceid=151, compaction requested=false 2024-12-10T00:16:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-10T00:16:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-10T00:16:20,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T00:16:20,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5920 sec 2024-12-10T00:16:20,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5960 sec 2024-12-10T00:16:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:21,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:21,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:21,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/5ec241bbf02c4cb095706a70cfcd3615 is 50, key is test_row_0/A:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742290_1466 (size=14541) 2024-12-10T00:16:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789841157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789781153, max timestamp=1733789781169Num uniques 859; 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789781153, max timestamp=1733789781169Num uniques 859; 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789781153, max timestamp=1733789781169Num uniques 859; 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:21,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789841199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789841202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789841203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789841204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789841280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789841309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789841319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789841319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789841320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-10T00:16:21,379 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:21,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-10T00:16:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:21,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:21,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:21,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789841489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789841520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789841530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789841530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789841530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/5ec241bbf02c4cb095706a70cfcd3615 2024-12-10T00:16:21,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/64e37e4042464a2698bbacad1fda3676 is 50, key is test_row_0/B:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:21,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742291_1467 (size=12151) 2024-12-10T00:16:21,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789841799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789841840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789841840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789841840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789841840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:21,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/64e37e4042464a2698bbacad1fda3676 2024-12-10T00:16:21,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7bd2535f44d343f79a8bcea6f0ba7fa4 is 50, key is test_row_0/C:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:21,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742292_1468 (size=12151) 2024-12-10T00:16:21,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:21,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:22,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789842320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:22,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789842360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789842360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789842360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:22,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7bd2535f44d343f79a8bcea6f0ba7fa4 2024-12-10T00:16:22,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789842360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:22,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/5ec241bbf02c4cb095706a70cfcd3615 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615 2024-12-10T00:16:22,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615, entries=200, sequenceid=164, filesize=14.2 K 2024-12-10T00:16:22,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/64e37e4042464a2698bbacad1fda3676 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676 2024-12-10T00:16:22,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676, entries=150, sequenceid=164, filesize=11.9 K 2024-12-10T00:16:22,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7bd2535f44d343f79a8bcea6f0ba7fa4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4 2024-12-10T00:16:22,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4, entries=150, sequenceid=164, filesize=11.9 K 2024-12-10T00:16:22,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ee7312073a8c1deb0dc9c8e31f81137a in 1256ms, sequenceid=164, compaction requested=true 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:22,384 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:22,384 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:22,385 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:22,385 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:22,385 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,385 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/677bb94f0f6245b2a2fb2e610f9462e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=38.1 K 2024-12-10T00:16:22,387 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:22,387 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677bb94f0f6245b2a2fb2e610f9462e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777745 2024-12-10T00:16:22,387 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:22,387 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,388 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/47aa147d16ce436baab4108cb774294d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.8 K 2024-12-10T00:16:22,388 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef50be2d79264f3091c738bf69121675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733789778946 2024-12-10T00:16:22,388 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 47aa147d16ce436baab4108cb774294d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777745 2024-12-10T00:16:22,388 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec241bbf02c4cb095706a70cfcd3615, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781120 2024-12-10T00:16:22,388 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2041307ca9054fea9bf1c027ea398869, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733789778946 2024-12-10T00:16:22,389 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e37e4042464a2698bbacad1fda3676, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781125 2024-12-10T00:16:22,394 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:22,395 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/dd821c4fd3a9440e99e92584c43f1eb5 is 50, key is test_row_0/A:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:22,398 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:22,399 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/662cbf48b77f48248aa06691c0f7da60 is 50, key is test_row_0/B:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:22,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742293_1469 (size=12561) 2024-12-10T00:16:22,407 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/dd821c4fd3a9440e99e92584c43f1eb5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/dd821c4fd3a9440e99e92584c43f1eb5 2024-12-10T00:16:22,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742294_1470 (size=12561) 2024-12-10T00:16:22,410 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into dd821c4fd3a9440e99e92584c43f1eb5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:22,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:22,410 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789782384; duration=0sec 2024-12-10T00:16:22,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:22,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:22,410 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:22,411 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:22,411 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:22,411 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,411 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4e2f660f09684450ac922a52a6cd0724, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=35.8 K 2024-12-10T00:16:22,411 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2f660f09684450ac922a52a6cd0724, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733789777745 2024-12-10T00:16:22,412 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b1e96d25fc44ffdb40c1ac94b8dd532, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733789778946 2024-12-10T00:16:22,412 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bd2535f44d343f79a8bcea6f0ba7fa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781125 2024-12-10T00:16:22,419 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:22,419 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4faf7512055f467d8256ad1d0f539479 is 50, key is test_row_0/C:col10/1733789781127/Put/seqid=0 2024-12-10T00:16:22,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742295_1471 (size=12561) 2024-12-10T00:16:22,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-10T00:16:22,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:22,451 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ab00cbb543fa4f33ab2e4fa46fffcc63 is 50, key is test_row_0/A:col10/1733789781191/Put/seqid=0 2024-12-10T00:16:22,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742296_1472 (size=12151) 2024-12-10T00:16:22,462 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ab00cbb543fa4f33ab2e4fa46fffcc63 2024-12-10T00:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/643cdd283c1a443f8e7e099fd46b2651 is 50, key is test_row_0/B:col10/1733789781191/Put/seqid=0 2024-12-10T00:16:22,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742297_1473 (size=12151) 2024-12-10T00:16:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:22,814 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/662cbf48b77f48248aa06691c0f7da60 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/662cbf48b77f48248aa06691c0f7da60 2024-12-10T00:16:22,818 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into 662cbf48b77f48248aa06691c0f7da60(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:22,818 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:22,818 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789782384; duration=0sec 2024-12-10T00:16:22,818 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:22,818 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:22,835 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/4faf7512055f467d8256ad1d0f539479 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4faf7512055f467d8256ad1d0f539479 2024-12-10T00:16:22,838 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 4faf7512055f467d8256ad1d0f539479(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:22,838 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:22,838 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789782384; duration=0sec 2024-12-10T00:16:22,838 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:22,838 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:22,871 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/643cdd283c1a443f8e7e099fd46b2651 2024-12-10T00:16:22,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a96207b6548241619e836fc435d7412a is 50, key is test_row_0/C:col10/1733789781191/Put/seqid=0 2024-12-10T00:16:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742298_1474 (size=12151) 2024-12-10T00:16:23,282 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a96207b6548241619e836fc435d7412a 2024-12-10T00:16:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ab00cbb543fa4f33ab2e4fa46fffcc63 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63 2024-12-10T00:16:23,289 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T00:16:23,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/643cdd283c1a443f8e7e099fd46b2651 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651 2024-12-10T00:16:23,292 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T00:16:23,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a96207b6548241619e836fc435d7412a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a 2024-12-10T00:16:23,297 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a, entries=150, sequenceid=199, filesize=11.9 K 2024-12-10T00:16:23,298 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for ee7312073a8c1deb0dc9c8e31f81137a in 848ms, sequenceid=199, compaction requested=false 2024-12-10T00:16:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-10T00:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-10T00:16:23,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-10T00:16:23,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-12-10T00:16:23,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.9240 sec 2024-12-10T00:16:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:23,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e is 50, key is test_row_0/A:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742299_1475 (size=14541) 2024-12-10T00:16:23,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e 2024-12-10T00:16:23,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/43e020997c3048968de677c0b5ea871a is 50, key is test_row_0/B:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742300_1476 (size=12151) 2024-12-10T00:16:23,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/43e020997c3048968de677c0b5ea871a 2024-12-10T00:16:23,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ec615b4903cb4efdb684fa4a6f13919a is 50, key is test_row_0/C:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789783395, max timestamp=1733789783408Num uniques 859; 2024-12-10T00:16:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789783395, max timestamp=1733789783408Num uniques 859; 2024-12-10T00:16:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:23,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789783395, max timestamp=1733789783408Num uniques 859; 2024-12-10T00:16:23,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742301_1477 (size=12151) 2024-12-10T00:16:23,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ec615b4903cb4efdb684fa4a6f13919a 2024-12-10T00:16:23,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e 2024-12-10T00:16:23,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e, entries=200, sequenceid=211, filesize=14.2 K 2024-12-10T00:16:23,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/43e020997c3048968de677c0b5ea871a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a 2024-12-10T00:16:23,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T00:16:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ec615b4903cb4efdb684fa4a6f13919a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a 2024-12-10T00:16:23,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a, entries=150, sequenceid=211, filesize=11.9 K 2024-12-10T00:16:23,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=87.22 KB/89310 for ee7312073a8c1deb0dc9c8e31f81137a in 71ms, sequenceid=211, compaction requested=true 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:23,434 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:23,434 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:23,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:23,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=93.93 KB heapSize=260.77 KB 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:23,435 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:23,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:23,435 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:23,435 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,435 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/dd821c4fd3a9440e99e92584c43f1eb5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=38.3 K 2024-12-10T00:16:23,435 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:23,435 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:23,435 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,435 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/662cbf48b77f48248aa06691c0f7da60, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.0 K 2024-12-10T00:16:23,436 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd821c4fd3a9440e99e92584c43f1eb5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781125 2024-12-10T00:16:23,436 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 662cbf48b77f48248aa06691c0f7da60, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781125 2024-12-10T00:16:23,436 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 643cdd283c1a443f8e7e099fd46b2651, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733789781191 2024-12-10T00:16:23,436 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab00cbb543fa4f33ab2e4fa46fffcc63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733789781191 2024-12-10T00:16:23,437 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e020997c3048968de677c0b5ea871a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783349 2024-12-10T00:16:23,437 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0c74aa74ec4fbeb7a765ff3d6ee62e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783340 2024-12-10T00:16:23,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6e0f2003c03947beb2f20005ed21913c is 50, key is test_row_0/A:col10/1733789783428/Put/seqid=0 2024-12-10T00:16:23,443 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:23,444 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/23e261253a70459591c139994e557950 is 50, key is test_row_0/B:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,451 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:23,451 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/967fd4740fdb4776973b06878536a385 is 50, key is test_row_0/A:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742303_1479 (size=12663) 2024-12-10T00:16:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742302_1478 (size=19321) 2024-12-10T00:16:23,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742304_1480 (size=12663) 2024-12-10T00:16:23,467 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/23e261253a70459591c139994e557950 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/23e261253a70459591c139994e557950 2024-12-10T00:16:23,471 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into 23e261253a70459591c139994e557950(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:23,472 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,472 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789783434; duration=0sec 2024-12-10T00:16:23,472 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:23,472 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:23,472 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:23,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:23,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:23,477 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,477 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4faf7512055f467d8256ad1d0f539479, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.0 K 2024-12-10T00:16:23,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4faf7512055f467d8256ad1d0f539479, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733789781125 2024-12-10T00:16:23,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a96207b6548241619e836fc435d7412a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733789781191 2024-12-10T00:16:23,478 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ec615b4903cb4efdb684fa4a6f13919a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783349 2024-12-10T00:16:23,482 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:23,483 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/fbd8979adf5145bc8350e328f2ba8663 is 50, key is test_row_0/C:col10/1733789783362/Put/seqid=0 2024-12-10T00:16:23,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742305_1481 (size=12663) 2024-12-10T00:16:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789843470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789843471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789843472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789843490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789843490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T00:16:23,530 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:23,530 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-10T00:16:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:23,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:23,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:23,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789843602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789843602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789843602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789843611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789843612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T00:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789843820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789843820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789843820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T00:16:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:23,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:23,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789843830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789843830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:23,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6e0f2003c03947beb2f20005ed21913c 2024-12-10T00:16:23,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/df699d53ab55497493e9c96f9aa1be36 is 50, key is test_row_0/B:col10/1733789783428/Put/seqid=0 2024-12-10T00:16:23,874 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/967fd4740fdb4776973b06878536a385 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/967fd4740fdb4776973b06878536a385 2024-12-10T00:16:23,880 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 967fd4740fdb4776973b06878536a385(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:23,880 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,880 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789783434; duration=0sec 2024-12-10T00:16:23,880 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:23,880 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:23,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742306_1482 (size=12151) 2024-12-10T00:16:23,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/df699d53ab55497493e9c96f9aa1be36 2024-12-10T00:16:23,890 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/fbd8979adf5145bc8350e328f2ba8663 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fbd8979adf5145bc8350e328f2ba8663 2024-12-10T00:16:23,895 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into fbd8979adf5145bc8350e328f2ba8663(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:23,895 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,895 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789783434; duration=0sec 2024-12-10T00:16:23,895 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:23,895 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:23,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/3c38311edc9e476a96796e12db5b3eef is 50, key is test_row_0/C:col10/1733789783428/Put/seqid=0 2024-12-10T00:16:23,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742307_1483 (size=12151) 2024-12-10T00:16:23,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/3c38311edc9e476a96796e12db5b3eef 2024-12-10T00:16:23,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/6e0f2003c03947beb2f20005ed21913c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c 2024-12-10T00:16:23,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c, entries=300, sequenceid=241, filesize=18.9 K 2024-12-10T00:16:23,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/df699d53ab55497493e9c96f9aa1be36 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36 2024-12-10T00:16:23,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T00:16:23,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/3c38311edc9e476a96796e12db5b3eef as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef 2024-12-10T00:16:23,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef, entries=150, sequenceid=241, filesize=11.9 K 2024-12-10T00:16:23,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~260.72 KB/266976, currentSize=93.93 KB/96180 for ee7312073a8c1deb0dc9c8e31f81137a in 492ms, sequenceid=241, compaction requested=false 2024-12-10T00:16:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:23,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:23,993 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:23,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/326161c0fdbd43249230613cfe1950f9 is 50, key is test_row_0/A:col10/1733789783471/Put/seqid=0 2024-12-10T00:16:24,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742308_1484 (size=12251) 2024-12-10T00:16:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:24,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:24,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789844151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789844152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789844156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789844157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789844158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789844269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789844269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789844270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789844270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789844270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,411 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/326161c0fdbd43249230613cfe1950f9 2024-12-10T00:16:24,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/edbd94879a1f4c44a81d06c7d31ccb27 is 50, key is test_row_0/B:col10/1733789783471/Put/seqid=0 2024-12-10T00:16:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742309_1485 (size=12251) 2024-12-10T00:16:24,423 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/edbd94879a1f4c44a81d06c7d31ccb27 2024-12-10T00:16:24,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/70e0a916976647c294fc1b2d3b86ace2 is 50, key is test_row_0/C:col10/1733789783471/Put/seqid=0 2024-12-10T00:16:24,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742310_1486 (size=12251) 2024-12-10T00:16:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789844790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:24,834 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/70e0a916976647c294fc1b2d3b86ace2 2024-12-10T00:16:24,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/326161c0fdbd43249230613cfe1950f9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9 2024-12-10T00:16:24,839 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9, entries=150, sequenceid=261, filesize=12.0 K 2024-12-10T00:16:24,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/edbd94879a1f4c44a81d06c7d31ccb27 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27 2024-12-10T00:16:24,843 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27, entries=150, sequenceid=261, filesize=12.0 K 2024-12-10T00:16:24,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/70e0a916976647c294fc1b2d3b86ace2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2 2024-12-10T00:16:24,846 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2, entries=150, sequenceid=261, filesize=12.0 K 2024-12-10T00:16:24,846 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ee7312073a8c1deb0dc9c8e31f81137a in 853ms, sequenceid=261, compaction requested=true 2024-12-10T00:16:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:24,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-10T00:16:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-10T00:16:24,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-10T00:16:24,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3160 sec 2024-12-10T00:16:24,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3200 sec 2024-12-10T00:16:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:25,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/804fae42d78742cd97f784db51c2cbe6 is 50, key is test_row_0/A:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742311_1487 (size=14741) 2024-12-10T00:16:25,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/804fae42d78742cd97f784db51c2cbe6 2024-12-10T00:16:25,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/126913faa9ee4f59962c8f9503954235 is 50, key is test_row_0/B:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742312_1488 (size=12301) 2024-12-10T00:16:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789845335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789845335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789845336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789845335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789845338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789845460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789845669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789845669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789845669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789845669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789845669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-10T00:16:25,680 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:25,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-10T00:16:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T00:16:25,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:25,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:25,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:25,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/126913faa9ee4f59962c8f9503954235 2024-12-10T00:16:25,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/6792aab36985465c8c3e5c4c651fb345 is 50, key is test_row_0/C:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742313_1489 (size=12301) 2024-12-10T00:16:25,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/6792aab36985465c8c3e5c4c651fb345 2024-12-10T00:16:25,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/804fae42d78742cd97f784db51c2cbe6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6 2024-12-10T00:16:25,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6, entries=200, sequenceid=277, filesize=14.4 K 2024-12-10T00:16:25,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/126913faa9ee4f59962c8f9503954235 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235 2024-12-10T00:16:25,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235, entries=150, sequenceid=277, filesize=12.0 K 2024-12-10T00:16:25,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/6792aab36985465c8c3e5c4c651fb345 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345 2024-12-10T00:16:25,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345, entries=150, sequenceid=277, filesize=12.0 K 2024-12-10T00:16:25,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ee7312073a8c1deb0dc9c8e31f81137a in 461ms, sequenceid=277, compaction requested=true 2024-12-10T00:16:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:25,763 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:25,763 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:25,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:25,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:25,764 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58976 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:25,764 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:25,764 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:25,764 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/967fd4740fdb4776973b06878536a385, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=57.6 K 2024-12-10T00:16:25,764 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:25,765 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:25,765 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/23e261253a70459591c139994e557950, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=48.2 K 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967fd4740fdb4776973b06878536a385, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783349 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e261253a70459591c139994e557950, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783349 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0f2003c03947beb2f20005ed21913c, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789783418 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting df699d53ab55497493e9c96f9aa1be36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789783420 2024-12-10T00:16:25,765 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326161c0fdbd43249230613cfe1950f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789783471 2024-12-10T00:16:25,766 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting edbd94879a1f4c44a81d06c7d31ccb27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789783471 2024-12-10T00:16:25,766 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 804fae42d78742cd97f784db51c2cbe6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784151 2024-12-10T00:16:25,766 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 126913faa9ee4f59962c8f9503954235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784155 2024-12-10T00:16:25,775 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:25,775 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:25,775 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/a091b1a17ec84f12a7ab263cda6402ae is 50, key is test_row_0/B:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,775 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d1f1a30fb483456997646ca99dcc41ac is 50, key is test_row_0/A:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T00:16:25,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742314_1490 (size=12949) 2024-12-10T00:16:25,795 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/a091b1a17ec84f12a7ab263cda6402ae as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a091b1a17ec84f12a7ab263cda6402ae 2024-12-10T00:16:25,800 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into a091b1a17ec84f12a7ab263cda6402ae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:25,801 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:25,801 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=12, startTime=1733789785763; duration=0sec 2024-12-10T00:16:25,801 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:25,801 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:25,801 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:25,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742315_1491 (size=12949) 2024-12-10T00:16:25,802 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:25,802 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:25,802 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:25,803 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fbd8979adf5145bc8350e328f2ba8663, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=48.2 K 2024-12-10T00:16:25,803 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd8979adf5145bc8350e328f2ba8663, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733789783349 2024-12-10T00:16:25,805 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c38311edc9e476a96796e12db5b3eef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733789783420 2024-12-10T00:16:25,806 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e0a916976647c294fc1b2d3b86ace2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733789783471 2024-12-10T00:16:25,807 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6792aab36985465c8c3e5c4c651fb345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784155 2024-12-10T00:16:25,809 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d1f1a30fb483456997646ca99dcc41ac as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1f1a30fb483456997646ca99dcc41ac 2024-12-10T00:16:25,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into d1f1a30fb483456997646ca99dcc41ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:25,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:25,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=12, startTime=1733789785763; duration=0sec 2024-12-10T00:16:25,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:25,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:25,821 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:25,821 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/5f4e2410b8c44e2783574ed45e1f2871 is 50, key is test_row_0/C:col10/1733789784155/Put/seqid=0 2024-12-10T00:16:25,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742316_1492 (size=12949) 2024-12-10T00:16:25,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:25,836 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:25,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d4d9c754b8f94c048ac7ee6aadb46ec7 is 50, key is test_row_0/A:col10/1733789785334/Put/seqid=0 2024-12-10T00:16:25,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742317_1493 (size=12301) 2024-12-10T00:16:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:25,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T00:16:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789846009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789846009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789846009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789846010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789846016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789846120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789846120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789846129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789846129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789846129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,233 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/5f4e2410b8c44e2783574ed45e1f2871 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5f4e2410b8c44e2783574ed45e1f2871 2024-12-10T00:16:26,237 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 5f4e2410b8c44e2783574ed45e1f2871(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:26,237 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:26,237 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=12, startTime=1733789785763; duration=0sec 2024-12-10T00:16:26,237 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:26,237 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:26,262 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d4d9c754b8f94c048ac7ee6aadb46ec7 2024-12-10T00:16:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/7b559da1b7974673a285af02e08b5c6e is 50, key is test_row_0/B:col10/1733789785334/Put/seqid=0 2024-12-10T00:16:26,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742318_1494 (size=12301) 2024-12-10T00:16:26,274 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/7b559da1b7974673a285af02e08b5c6e 2024-12-10T00:16:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0eea0647a70c4c44812144d7b865801f is 50, key is test_row_0/C:col10/1733789785334/Put/seqid=0 2024-12-10T00:16:26,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742319_1495 (size=12301) 2024-12-10T00:16:26,289 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0eea0647a70c4c44812144d7b865801f 2024-12-10T00:16:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/d4d9c754b8f94c048ac7ee6aadb46ec7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7 2024-12-10T00:16:26,296 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T00:16:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/7b559da1b7974673a285af02e08b5c6e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e 2024-12-10T00:16:26,302 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T00:16:26,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0eea0647a70c4c44812144d7b865801f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f 2024-12-10T00:16:26,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T00:16:26,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ee7312073a8c1deb0dc9c8e31f81137a in 473ms, sequenceid=295, compaction requested=false 2024-12-10T00:16:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-10T00:16:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T00:16:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-10T00:16:26,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-10T00:16:26,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 627 msec 2024-12-10T00:16:26,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 633 msec 2024-12-10T00:16:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:26,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:26,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:26,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/efc934852db34d49893a0787b60747b9 is 50, key is test_row_0/A:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742320_1496 (size=12301) 2024-12-10T00:16:26,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/efc934852db34d49893a0787b60747b9 2024-12-10T00:16:26,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/4aa754f687df44e9ac8a1d9a5855d9e3 is 50, key is test_row_0/B:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:26,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742321_1497 (size=12301) 2024-12-10T00:16:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789846422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789846422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789846434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789846435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789846436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789846539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789846539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789846550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789846550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789846550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789846749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789846760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789846760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789846760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789846760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:26,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/4aa754f687df44e9ac8a1d9a5855d9e3 2024-12-10T00:16:26,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a87fd85dbb494440828c1c30a6018b90 is 50, key is test_row_0/C:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742322_1498 (size=12301) 2024-12-10T00:16:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-10T00:16:26,820 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:26,820 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-10T00:16:26,822 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:26,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:26,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:26,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-10T00:16:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:26,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789847060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789847070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789847070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789847070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789847070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-10T00:16:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:27,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a87fd85dbb494440828c1c30a6018b90 2024-12-10T00:16:27,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/efc934852db34d49893a0787b60747b9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9 2024-12-10T00:16:27,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T00:16:27,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/4aa754f687df44e9ac8a1d9a5855d9e3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3 2024-12-10T00:16:27,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T00:16:27,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a87fd85dbb494440828c1c30a6018b90 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90 2024-12-10T00:16:27,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90, entries=150, sequenceid=315, filesize=12.0 K 2024-12-10T00:16:27,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ee7312073a8c1deb0dc9c8e31f81137a in 850ms, sequenceid=315, compaction requested=true 2024-12-10T00:16:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:27,227 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:27,227 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:27,227 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:27,227 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:27,228 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,228 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,228 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a091b1a17ec84f12a7ab263cda6402ae, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.7 K 2024-12-10T00:16:27,228 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1f1a30fb483456997646ca99dcc41ac, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.7 K 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a091b1a17ec84f12a7ab263cda6402ae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784155 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f1a30fb483456997646ca99dcc41ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784155 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b559da1b7974673a285af02e08b5c6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733789785334 2024-12-10T00:16:27,228 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4d9c754b8f94c048ac7ee6aadb46ec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733789785334 2024-12-10T00:16:27,229 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa754f687df44e9ac8a1d9a5855d9e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:27,229 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc934852db34d49893a0787b60747b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:27,243 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:27,243 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/e007a9fbcbe1470ebca4bbd2b9d0eeba is 50, key is test_row_0/B:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:27,250 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:27,250 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/10073eb681524117ab02d5a02e9b160f is 50, key is test_row_0/A:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:27,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742323_1499 (size=13051) 2024-12-10T00:16:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742324_1500 (size=13051) 2024-12-10T00:16:27,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-10T00:16:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/7dab1cb040f0464894144c511165a0c9 is 50, key is test_row_0/A:col10/1733789786432/Put/seqid=0 2024-12-10T00:16:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742325_1501 (size=12301) 2024-12-10T00:16:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:27,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789847603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789847604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789847605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789847613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789847615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,661 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/e007a9fbcbe1470ebca4bbd2b9d0eeba as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/e007a9fbcbe1470ebca4bbd2b9d0eeba 2024-12-10T00:16:27,665 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into e007a9fbcbe1470ebca4bbd2b9d0eeba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:27,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:27,666 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789787227; duration=0sec 2024-12-10T00:16:27,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:27,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:27,666 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:27,667 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:27,667 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:27,667 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:27,667 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5f4e2410b8c44e2783574ed45e1f2871, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.7 K 2024-12-10T00:16:27,667 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4e2410b8c44e2783574ed45e1f2871, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733789784155 2024-12-10T00:16:27,667 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eea0647a70c4c44812144d7b865801f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733789785334 2024-12-10T00:16:27,668 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a87fd85dbb494440828c1c30a6018b90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:27,676 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/10073eb681524117ab02d5a02e9b160f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/10073eb681524117ab02d5a02e9b160f 2024-12-10T00:16:27,678 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:27,679 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a0c4f669556349b3bdd062e3c77ccb29 is 50, key is test_row_0/C:col10/1733789786359/Put/seqid=0 2024-12-10T00:16:27,681 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 10073eb681524117ab02d5a02e9b160f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:27,681 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:27,682 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789787226; duration=0sec 2024-12-10T00:16:27,682 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:27,682 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742326_1502 (size=13051) 2024-12-10T00:16:27,689 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/7dab1cb040f0464894144c511165a0c9 2024-12-10T00:16:27,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/39151fe9ff3942ad872fb2f3a345d3f2 is 50, key is test_row_0/B:col10/1733789786432/Put/seqid=0 2024-12-10T00:16:27,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742327_1503 (size=12301) 2024-12-10T00:16:27,703 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/39151fe9ff3942ad872fb2f3a345d3f2 2024-12-10T00:16:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0dfdcb98ceaa44aab9bb98dadadfffd9 is 50, key is test_row_0/C:col10/1733789786432/Put/seqid=0 2024-12-10T00:16:27,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742328_1504 (size=12301) 2024-12-10T00:16:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789847719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789847719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789847720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789847730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789847730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789847929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789847939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789847940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789847940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:27,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789847940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:28,087 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/a0c4f669556349b3bdd062e3c77ccb29 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a0c4f669556349b3bdd062e3c77ccb29 2024-12-10T00:16:28,091 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into a0c4f669556349b3bdd062e3c77ccb29(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:28,091 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:28,091 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789787227; duration=0sec 2024-12-10T00:16:28,091 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:28,091 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:28,119 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0dfdcb98ceaa44aab9bb98dadadfffd9 2024-12-10T00:16:28,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/7dab1cb040f0464894144c511165a0c9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9 2024-12-10T00:16:28,126 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9, entries=150, sequenceid=328, filesize=12.0 K 2024-12-10T00:16:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/39151fe9ff3942ad872fb2f3a345d3f2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2 2024-12-10T00:16:28,130 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2, entries=150, sequenceid=328, filesize=12.0 K 2024-12-10T00:16:28,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/0dfdcb98ceaa44aab9bb98dadadfffd9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9 2024-12-10T00:16:28,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9, entries=150, sequenceid=328, filesize=12.0 K 2024-12-10T00:16:28,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ee7312073a8c1deb0dc9c8e31f81137a in 854ms, sequenceid=328, compaction requested=false 2024-12-10T00:16:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-10T00:16:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-10T00:16:28,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-10T00:16:28,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-12-10T00:16:28,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3190 sec 2024-12-10T00:16:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:28,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:28,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/21403acb58fd4b1880eecee088dfcf04 is 50, key is test_row_0/A:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:28,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742329_1505 (size=14741) 2024-12-10T00:16:28,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789848267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789848268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789848270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789848277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789848277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789848379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789848379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789848390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789848390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789848390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789848589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789848589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789848600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789848600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789848600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/21403acb58fd4b1880eecee088dfcf04 2024-12-10T00:16:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/f3b0668d8dc243cf990ed776bb9d122c is 50, key is test_row_0/B:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:28,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742330_1506 (size=12301) 2024-12-10T00:16:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789848899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789848899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789848909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789848910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789848910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-10T00:16:28,969 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:28,970 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-10T00:16:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T00:16:28,972 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:28,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:28,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T00:16:29,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/f3b0668d8dc243cf990ed776bb9d122c 2024-12-10T00:16:29,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/951c9d80b6c144be80aabbab635ae3c4 is 50, key is test_row_0/C:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:29,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742331_1507 (size=12301) 2024-12-10T00:16:29,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-10T00:16:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-10T00:16:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T00:16:29,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789849419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:29,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789849419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:29,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-10T00:16:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789849430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789849430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789849430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:29,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:29,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/951c9d80b6c144be80aabbab635ae3c4 2024-12-10T00:16:29,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/21403acb58fd4b1880eecee088dfcf04 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04 2024-12-10T00:16:29,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04, entries=200, sequenceid=351, filesize=14.4 K 2024-12-10T00:16:29,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/f3b0668d8dc243cf990ed776bb9d122c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c 2024-12-10T00:16:29,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T00:16:29,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/951c9d80b6c144be80aabbab635ae3c4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4 2024-12-10T00:16:29,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T00:16:29,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ee7312073a8c1deb0dc9c8e31f81137a in 1254ms, sequenceid=351, compaction requested=true 2024-12-10T00:16:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:29,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:29,508 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:29,508 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:29,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:29,510 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,510 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,510 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/10073eb681524117ab02d5a02e9b160f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=39.2 K 2024-12-10T00:16:29,510 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/e007a9fbcbe1470ebca4bbd2b9d0eeba, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.8 K 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10073eb681524117ab02d5a02e9b160f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:29,510 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e007a9fbcbe1470ebca4bbd2b9d0eeba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:29,511 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 39151fe9ff3942ad872fb2f3a345d3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733789786432 2024-12-10T00:16:29,511 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dab1cb040f0464894144c511165a0c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733789786432 2024-12-10T00:16:29,511 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b0668d8dc243cf990ed776bb9d122c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787603 2024-12-10T00:16:29,511 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21403acb58fd4b1880eecee088dfcf04, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787598 2024-12-10T00:16:29,517 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:29,517 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/92d371adc2274aaf9f534bf7ffe3eab5 is 50, key is test_row_0/A:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:29,517 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:29,518 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/a4c53ba8ff184f32aba357ccba7a9cc1 is 50, key is test_row_0/B:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742332_1508 (size=13153) 2024-12-10T00:16:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742333_1509 (size=13153) 2024-12-10T00:16:29,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,585 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/550538365302466096dc0ea32926197f is 50, key is test_row_0/A:col10/1733789788274/Put/seqid=0 2024-12-10T00:16:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742334_1510 (size=12301) 2024-12-10T00:16:29,598 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/550538365302466096dc0ea32926197f 2024-12-10T00:16:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T00:16:29,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/beed95b834e949af9e2cae6012e84c85 is 50, key is test_row_0/B:col10/1733789788274/Put/seqid=0 2024-12-10T00:16:29,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742335_1511 (size=12301) 2024-12-10T00:16:29,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/beed95b834e949af9e2cae6012e84c85 2024-12-10T00:16:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/b168bf7d62c2490ca107b09289894afd is 50, key is test_row_0/C:col10/1733789788274/Put/seqid=0 2024-12-10T00:16:29,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742336_1512 (size=12301) 2024-12-10T00:16:29,930 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/92d371adc2274aaf9f534bf7ffe3eab5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/92d371adc2274aaf9f534bf7ffe3eab5 2024-12-10T00:16:29,937 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 92d371adc2274aaf9f534bf7ffe3eab5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:29,937 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:29,937 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789789508; duration=0sec 2024-12-10T00:16:29,937 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:29,937 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:29,938 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:29,941 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:29,941 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:29,941 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:29,941 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a0c4f669556349b3bdd062e3c77ccb29, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.8 K 2024-12-10T00:16:29,945 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c4f669556349b3bdd062e3c77ccb29, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733789786007 2024-12-10T00:16:29,945 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dfdcb98ceaa44aab9bb98dadadfffd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733789786432 2024-12-10T00:16:29,946 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 951c9d80b6c144be80aabbab635ae3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787603 2024-12-10T00:16:29,946 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/a4c53ba8ff184f32aba357ccba7a9cc1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a4c53ba8ff184f32aba357ccba7a9cc1 2024-12-10T00:16:29,951 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into a4c53ba8ff184f32aba357ccba7a9cc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:29,951 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:29,951 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789789508; duration=0sec 2024-12-10T00:16:29,952 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:29,952 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:29,964 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:29,964 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/730a1bc2c35543abb61d41d90024fe6b is 50, key is test_row_0/C:col10/1733789787603/Put/seqid=0 2024-12-10T00:16:29,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742337_1513 (size=13153) 2024-12-10T00:16:29,983 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/730a1bc2c35543abb61d41d90024fe6b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/730a1bc2c35543abb61d41d90024fe6b 2024-12-10T00:16:29,989 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 730a1bc2c35543abb61d41d90024fe6b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:29,989 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:29,989 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789789509; duration=0sec 2024-12-10T00:16:29,989 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:29,989 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:30,036 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/b168bf7d62c2490ca107b09289894afd 2024-12-10T00:16:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/550538365302466096dc0ea32926197f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f 2024-12-10T00:16:30,043 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:16:30,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/beed95b834e949af9e2cae6012e84c85 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85 2024-12-10T00:16:30,048 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:16:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/b168bf7d62c2490ca107b09289894afd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd 2024-12-10T00:16:30,053 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd, entries=150, sequenceid=364, filesize=12.0 K 2024-12-10T00:16:30,054 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for ee7312073a8c1deb0dc9c8e31f81137a in 468ms, sequenceid=364, compaction requested=false 2024-12-10T00:16:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-10T00:16:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-10T00:16:30,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-10T00:16:30,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-12-10T00:16:30,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0920 sec 2024-12-10T00:16:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-10T00:16:30,110 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:30,110 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-10T00:16:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-10T00:16:30,112 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:30,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:30,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-10T00:16:30,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-10T00:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-10T00:16:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-10T00:16:30,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-10T00:16:30,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-10T00:16:30,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 159 msec 2024-12-10T00:16:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-10T00:16:30,430 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:30,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-12-10T00:16:30,431 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:30,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:30,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:30,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:30,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/132ce880b5a3430b859af884a6619d3d is 50, key is test_row_0/A:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742338_1514 (size=14741) 2024-12-10T00:16:30,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789850481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789790479, max timestamp=1733789790488Num uniques 859; 2024-12-10T00:16:30,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789790479, max timestamp=1733789790488Num uniques 859; 2024-12-10T00:16:30,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:30,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789850481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733789790479, max timestamp=1733789790488Num uniques 859; 2024-12-10T00:16:30,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T00:16:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789850532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789850531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789850535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789850599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789850599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789850649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789850649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789850649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789850809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789850809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/132ce880b5a3430b859af884a6619d3d 2024-12-10T00:16:30,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/9e19be0731234b31ae8a42de82e21584 is 50, key is test_row_0/B:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789850870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789850872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789850872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:30,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:30,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:30,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:30,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742339_1515 (size=12301) 2024-12-10T00:16:31,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789851130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789851130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789851190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789851190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789851190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/9e19be0731234b31ae8a42de82e21584 2024-12-10T00:16:31,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7959557c8a984d48b23f6413fce07970 is 50, key is test_row_0/C:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742340_1516 (size=12301) 2024-12-10T00:16:31,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7959557c8a984d48b23f6413fce07970 2024-12-10T00:16:31,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/132ce880b5a3430b859af884a6619d3d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d 2024-12-10T00:16:31,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d, entries=200, sequenceid=378, filesize=14.4 K 2024-12-10T00:16:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/9e19be0731234b31ae8a42de82e21584 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584 2024-12-10T00:16:31,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. as already flushing 2024-12-10T00:16:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584, entries=150, sequenceid=378, filesize=12.0 K 2024-12-10T00:16:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:31,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7959557c8a984d48b23f6413fce07970 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970 2024-12-10T00:16:31,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970, entries=150, sequenceid=378, filesize=12.0 K 2024-12-10T00:16:31,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee7312073a8c1deb0dc9c8e31f81137a in 917ms, sequenceid=378, compaction requested=true 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:31,366 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:31,366 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:31,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:31,367 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:31,367 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:31,367 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:31,367 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:31,367 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,367 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,367 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a4c53ba8ff184f32aba357ccba7a9cc1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.9 K 2024-12-10T00:16:31,367 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/92d371adc2274aaf9f534bf7ffe3eab5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=39.3 K 2024-12-10T00:16:31,367 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d371adc2274aaf9f534bf7ffe3eab5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787603 2024-12-10T00:16:31,368 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c53ba8ff184f32aba357ccba7a9cc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787603 2024-12-10T00:16:31,368 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 550538365302466096dc0ea32926197f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789788266 2024-12-10T00:16:31,368 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132ce880b5a3430b859af884a6619d3d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790442 2024-12-10T00:16:31,368 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting beed95b834e949af9e2cae6012e84c85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789788266 2024-12-10T00:16:31,368 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e19be0731234b31ae8a42de82e21584, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790445 2024-12-10T00:16:31,407 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:31,407 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/a015b92f06ba41e8965c2e8815bc1881 is 50, key is test_row_0/A:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:31,409 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:31,410 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/5246e130bbe1411ca3e31867f1e4f0b6 is 50, key is test_row_0/B:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:31,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742342_1518 (size=13255) 2024-12-10T00:16:31,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742341_1517 (size=13255) 2024-12-10T00:16:31,434 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/5246e130bbe1411ca3e31867f1e4f0b6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/5246e130bbe1411ca3e31867f1e4f0b6 2024-12-10T00:16:31,438 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into 5246e130bbe1411ca3e31867f1e4f0b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:31,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:31,438 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789791366; duration=0sec 2024-12-10T00:16:31,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:31,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:31,438 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:31,439 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:31,439 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:31,439 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,439 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/730a1bc2c35543abb61d41d90024fe6b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=36.9 K 2024-12-10T00:16:31,439 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 730a1bc2c35543abb61d41d90024fe6b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733789787603 2024-12-10T00:16:31,441 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b168bf7d62c2490ca107b09289894afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733789788266 2024-12-10T00:16:31,441 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7959557c8a984d48b23f6413fce07970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790445 2024-12-10T00:16:31,446 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:31,446 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ee77e4f79baa47d3884555d88a00f8f1 is 50, key is test_row_0/C:col10/1733789790445/Put/seqid=0 2024-12-10T00:16:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742343_1519 (size=13255) 2024-12-10T00:16:31,458 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/ee77e4f79baa47d3884555d88a00f8f1 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ee77e4f79baa47d3884555d88a00f8f1 2024-12-10T00:16:31,461 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into ee77e4f79baa47d3884555d88a00f8f1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:31,461 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:31,461 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789791366; duration=0sec 2024-12-10T00:16:31,461 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:31,461 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:31,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-10T00:16:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,513 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:31,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/4720484840b242e383f20874c9b71939 is 50, key is test_row_0/A:col10/1733789790530/Put/seqid=0 2024-12-10T00:16:31,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742344_1520 (size=12301) 2024-12-10T00:16:31,522 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/4720484840b242e383f20874c9b71939 2024-12-10T00:16:31,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fc14d5709bf04daaa01f48dac148595f is 50, key is test_row_0/B:col10/1733789790530/Put/seqid=0 2024-12-10T00:16:31,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742345_1521 (size=12301) 2024-12-10T00:16:31,556 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fc14d5709bf04daaa01f48dac148595f 2024-12-10T00:16:31,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/01992941690a411ba2fe7c1169afbd36 is 50, key is test_row_0/C:col10/1733789790530/Put/seqid=0 2024-12-10T00:16:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:31,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742346_1522 (size=12301) 2024-12-10T00:16:31,600 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/01992941690a411ba2fe7c1169afbd36 2024-12-10T00:16:31,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/4720484840b242e383f20874c9b71939 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939 2024-12-10T00:16:31,607 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939, entries=150, sequenceid=416, filesize=12.0 K 2024-12-10T00:16:31,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/fc14d5709bf04daaa01f48dac148595f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f 2024-12-10T00:16:31,615 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f, entries=150, sequenceid=416, filesize=12.0 K 2024-12-10T00:16:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/01992941690a411ba2fe7c1169afbd36 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36 2024-12-10T00:16:31,620 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36, entries=150, sequenceid=416, filesize=12.0 K 2024-12-10T00:16:31,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for ee7312073a8c1deb0dc9c8e31f81137a in 108ms, sequenceid=416, compaction requested=false 2024-12-10T00:16:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-10T00:16:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-10T00:16:31,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-10T00:16:31,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1910 sec 2024-12-10T00:16:31,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.1950 sec 2024-12-10T00:16:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:31,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:31,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:31,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/54b78bfc665c4ad382eed9021b2cea0b is 50, key is test_row_0/A:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:31,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742347_1523 (size=17181) 2024-12-10T00:16:31,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/54b78bfc665c4ad382eed9021b2cea0b 2024-12-10T00:16:31,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/414e34a211b241c4b963380a1e01dbe5 is 50, key is test_row_0/B:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:31,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742348_1524 (size=12301) 2024-12-10T00:16:31,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/414e34a211b241c4b963380a1e01dbe5 2024-12-10T00:16:31,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7b4d6ca648fd4af6a480a3524b10f0cf is 50, key is test_row_0/C:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:31,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742349_1525 (size=12301) 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789791704, max timestamp=1733789791736Num uniques 859; 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789791704, max timestamp=1733789791736Num uniques 859; 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789791704, max timestamp=1733789791736Num uniques 859; 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:31,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789851758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789851759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789851764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789851764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789851772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,783 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-12-10T00:16:31,783 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:31,783 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,783 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:31,789 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-12-10T00:16:31,789 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:31,789 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,789 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:31,790 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-12-10T00:16:31,790 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:31,790 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,790 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:31,793 INFO [Thread-1961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1961. 2024-12-10T00:16:31,793 DEBUG [Thread-1961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:31,793 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,793 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:31,798 INFO [Thread-1963 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1963. 2024-12-10T00:16:31,798 DEBUG [Thread-1963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:31,798 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,798 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:31,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:31,833 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/a015b92f06ba41e8965c2e8815bc1881 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/a015b92f06ba41e8965c2e8815bc1881 2024-12-10T00:16:31,837 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into a015b92f06ba41e8965c2e8815bc1881(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:31,837 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:31,837 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789791366; duration=0sec 2024-12-10T00:16:31,837 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:31,837 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789851880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789851880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789851880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789851880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789851880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:31,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43626 deadline: 1733789852090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43668 deadline: 1733789852090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1733789852090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43586 deadline: 1733789852090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733789852090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 , the old value is region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., hostname=a67c4886b4f7,36911,1733789645811, seqNum=2 because the exception is null or not the one we care about 2024-12-10T00:16:32,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7b4d6ca648fd4af6a480a3524b10f0cf 2024-12-10T00:16:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/54b78bfc665c4ad382eed9021b2cea0b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b 2024-12-10T00:16:32,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b, entries=250, sequenceid=426, filesize=16.8 K 2024-12-10T00:16:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/414e34a211b241c4b963380a1e01dbe5 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5 2024-12-10T00:16:32,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5, entries=150, sequenceid=426, filesize=12.0 K 2024-12-10T00:16:32,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/7b4d6ca648fd4af6a480a3524b10f0cf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf 2024-12-10T00:16:32,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf, entries=150, sequenceid=426, filesize=12.0 K 2024-12-10T00:16:32,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee7312073a8c1deb0dc9c8e31f81137a in 474ms, sequenceid=426, compaction requested=true 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:32,128 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee7312073a8c1deb0dc9c8e31f81137a:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:32,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:32,128 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/B is initiating minor compaction (all files) 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/A is initiating minor compaction (all files) 2024-12-10T00:16:32,129 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/B in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:32,129 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/A in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:32,129 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/5246e130bbe1411ca3e31867f1e4f0b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=37.0 K 2024-12-10T00:16:32,129 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/a015b92f06ba41e8965c2e8815bc1881, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=41.7 K 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a015b92f06ba41e8965c2e8815bc1881, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790445 2024-12-10T00:16:32,129 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5246e130bbe1411ca3e31867f1e4f0b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790445 2024-12-10T00:16:32,130 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4720484840b242e383f20874c9b71939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789790529 2024-12-10T00:16:32,130 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fc14d5709bf04daaa01f48dac148595f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789790529 2024-12-10T00:16:32,130 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 414e34a211b241c4b963380a1e01dbe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733789791647 2024-12-10T00:16:32,130 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b78bfc665c4ad382eed9021b2cea0b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733789791642 2024-12-10T00:16:32,139 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#B#compaction#454 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:32,139 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/c2e1c2ff59434716bc88d8f6c6a82eee is 50, key is test_row_0/B:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:32,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742350_1526 (size=13357) 2024-12-10T00:16:32,147 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#A#compaction#455 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:32,147 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/9fa3a91de2ae4a16ba4fc4ac32e53bb3 is 50, key is test_row_0/A:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:32,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742351_1527 (size=13357) 2024-12-10T00:16:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:32,402 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-12-10T00:16:32,402 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-12-10T00:16:32,402 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:32,402 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:32,402 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,402 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,402 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,402 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee7312073a8c1deb0dc9c8e31f81137a 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-12-10T00:16:32,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,403 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-12-10T00:16:32,403 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:32,403 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,404 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733789791762, max timestamp=1733789792400Num uniques 1118; 2024-12-10T00:16:32,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733789791762, max timestamp=1733789792400Num uniques 1118; 2024-12-10T00:16:32,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-12-10T00:16:32,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=A 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=B 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee7312073a8c1deb0dc9c8e31f81137a, store=C 2024-12-10T00:16:32,404 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-12-10T00:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:32,404 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:32,404 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,405 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,405 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-12-10T00:16:32,405 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:32,405 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,405 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/0d0c7600122047fe9d7d6c15d677155e is 50, key is test_row_0/A:col10/1733789792402/Put/seqid=0 2024-12-10T00:16:32,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742352_1528 (size=12301) 2024-12-10T00:16:32,555 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/c2e1c2ff59434716bc88d8f6c6a82eee as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/c2e1c2ff59434716bc88d8f6c6a82eee 2024-12-10T00:16:32,563 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/B of ee7312073a8c1deb0dc9c8e31f81137a into c2e1c2ff59434716bc88d8f6c6a82eee(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:32,563 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:32,563 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/B, priority=13, startTime=1733789792128; duration=0sec 2024-12-10T00:16:32,563 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:32,563 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:B 2024-12-10T00:16:32,563 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:32,563 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/9fa3a91de2ae4a16ba4fc4ac32e53bb3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/9fa3a91de2ae4a16ba4fc4ac32e53bb3 2024-12-10T00:16:32,564 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:32,564 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): ee7312073a8c1deb0dc9c8e31f81137a/C is initiating minor compaction (all files) 2024-12-10T00:16:32,564 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee7312073a8c1deb0dc9c8e31f81137a/C in TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:32,564 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ee77e4f79baa47d3884555d88a00f8f1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp, totalSize=37.0 K 2024-12-10T00:16:32,565 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ee77e4f79baa47d3884555d88a00f8f1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733789790445 2024-12-10T00:16:32,565 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 01992941690a411ba2fe7c1169afbd36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733789790529 2024-12-10T00:16:32,565 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4d6ca648fd4af6a480a3524b10f0cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733789791647 2024-12-10T00:16:32,568 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/A of ee7312073a8c1deb0dc9c8e31f81137a into 9fa3a91de2ae4a16ba4fc4ac32e53bb3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:32,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:32,568 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/A, priority=13, startTime=1733789792128; duration=0sec 2024-12-10T00:16:32,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:32,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:A 2024-12-10T00:16:32,572 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee7312073a8c1deb0dc9c8e31f81137a#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:32,573 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/89e6941a25214858bff59ee775570d7a is 50, key is test_row_0/C:col10/1733789791652/Put/seqid=0 2024-12-10T00:16:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742353_1529 (size=13357) 2024-12-10T00:16:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-10T00:16:32,580 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1639 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4915 rows 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1643 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4926 rows 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1662 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4977 rows 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1641 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4920 rows 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1649 2024-12-10T00:16:32,581 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4943 rows 2024-12-10T00:16:32,581 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:16:32,581 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:16:32,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:32,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:32,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:16:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-10T00:16:32,586 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789792585"}]},"ts":"1733789792585"} 2024-12-10T00:16:32,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:16:32,587 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:16:32,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:16:32,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, UNASSIGN}] 2024-12-10T00:16:32,591 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, UNASSIGN 2024-12-10T00:16:32,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ee7312073a8c1deb0dc9c8e31f81137a, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:32,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, UNASSIGN because future has completed 2024-12-10T00:16:32,594 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:16:32,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-10T00:16:32,746 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:32,746 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:16:32,746 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing ee7312073a8c1deb0dc9c8e31f81137a, disabling compactions & flushes 2024-12-10T00:16:32,747 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 1 compactions & cache flush to complete for region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:32,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/0d0c7600122047fe9d7d6c15d677155e 2024-12-10T00:16:32,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/d1b0265444d342dcad5e76a235a16def is 50, key is test_row_0/B:col10/1733789792402/Put/seqid=0 2024-12-10T00:16:32,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742354_1530 (size=12301) 2024-12-10T00:16:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-10T00:16:32,984 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/89e6941a25214858bff59ee775570d7a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/89e6941a25214858bff59ee775570d7a 2024-12-10T00:16:32,993 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee7312073a8c1deb0dc9c8e31f81137a/C of ee7312073a8c1deb0dc9c8e31f81137a into 89e6941a25214858bff59ee775570d7a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:32,993 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:32,993 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a., storeName=ee7312073a8c1deb0dc9c8e31f81137a/C, priority=13, startTime=1733789792128; duration=0sec 2024-12-10T00:16:32,993 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:32,993 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:32,993 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee7312073a8c1deb0dc9c8e31f81137a:C 2024-12-10T00:16:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-10T00:16:33,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/d1b0265444d342dcad5e76a235a16def 2024-12-10T00:16:33,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/277d02e44e994929ac832068771860d6 is 50, key is test_row_0/C:col10/1733789792402/Put/seqid=0 2024-12-10T00:16:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742355_1531 (size=12301) 2024-12-10T00:16:33,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.62 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/277d02e44e994929ac832068771860d6 2024-12-10T00:16:33,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/A/0d0c7600122047fe9d7d6c15d677155e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/0d0c7600122047fe9d7d6c15d677155e 2024-12-10T00:16:33,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/0d0c7600122047fe9d7d6c15d677155e, entries=150, sequenceid=467, filesize=12.0 K 2024-12-10T00:16:33,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/B/d1b0265444d342dcad5e76a235a16def as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/d1b0265444d342dcad5e76a235a16def 2024-12-10T00:16:33,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/d1b0265444d342dcad5e76a235a16def, entries=150, sequenceid=467, filesize=12.0 K 2024-12-10T00:16:33,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/.tmp/C/277d02e44e994929ac832068771860d6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/277d02e44e994929ac832068771860d6 2024-12-10T00:16:33,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/277d02e44e994929ac832068771860d6, entries=150, sequenceid=467, filesize=12.0 K 2024-12-10T00:16:33,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~86.42 KB/88492, heapSize ~237.08 KB/242768, currentSize=0 B/0 for ee7312073a8c1deb0dc9c8e31f81137a in 1247ms, sequenceid=467, compaction requested=false 2024-12-10T00:16:33,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee7312073a8c1deb0dc9c8e31f81137a: 2024-12-10T00:16:33,649 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:33,649 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:33,649 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. after waiting 0 ms 2024-12-10T00:16:33,649 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:33,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/3f744356fe534771a88f79acb211b825, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/aba0521cbfb541ebbac170ce2ac35486, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/677bb94f0f6245b2a2fb2e610f9462e5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/dd821c4fd3a9440e99e92584c43f1eb5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/967fd4740fdb4776973b06878536a385, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1f1a30fb483456997646ca99dcc41ac, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/10073eb681524117ab02d5a02e9b160f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/92d371adc2274aaf9f534bf7ffe3eab5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/a015b92f06ba41e8965c2e8815bc1881, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b] to archive 2024-12-10T00:16:33,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1b5f498a96d42f68071ebccfe84a01c 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6b6fd69c7cdd4f6eb23876c8bee5f7d8 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/3f744356fe534771a88f79acb211b825 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/3f744356fe534771a88f79acb211b825 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/64b106a1c5e747569fd749504ca84e3f 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/aba0521cbfb541ebbac170ce2ac35486 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/aba0521cbfb541ebbac170ce2ac35486 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/68854c91e82948fbb48cf3e5348ef85a 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/be15651f834d42bc98cfc302670d03f4 2024-12-10T00:16:33,652 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/175b951879eb4daaaaacb43435a1e06b 2024-12-10T00:16:33,653 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/128e97b578ea4e69816f36ee53ac1f0c 2024-12-10T00:16:33,653 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/677bb94f0f6245b2a2fb2e610f9462e5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/677bb94f0f6245b2a2fb2e610f9462e5 2024-12-10T00:16:33,653 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ef50be2d79264f3091c738bf69121675 2024-12-10T00:16:33,653 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/5ec241bbf02c4cb095706a70cfcd3615 2024-12-10T00:16:33,654 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/dd821c4fd3a9440e99e92584c43f1eb5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/dd821c4fd3a9440e99e92584c43f1eb5 2024-12-10T00:16:33,654 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ab00cbb543fa4f33ab2e4fa46fffcc63 2024-12-10T00:16:33,654 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/967fd4740fdb4776973b06878536a385 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/967fd4740fdb4776973b06878536a385 2024-12-10T00:16:33,654 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/ad0c74aa74ec4fbeb7a765ff3d6ee62e 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/804fae42d78742cd97f784db51c2cbe6 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/326161c0fdbd43249230613cfe1950f9 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1f1a30fb483456997646ca99dcc41ac to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d1f1a30fb483456997646ca99dcc41ac 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/10073eb681524117ab02d5a02e9b160f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/10073eb681524117ab02d5a02e9b160f 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/d4d9c754b8f94c048ac7ee6aadb46ec7 2024-12-10T00:16:33,655 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/efc934852db34d49893a0787b60747b9 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/6e0f2003c03947beb2f20005ed21913c 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/21403acb58fd4b1880eecee088dfcf04 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/92d371adc2274aaf9f534bf7ffe3eab5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/92d371adc2274aaf9f534bf7ffe3eab5 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/550538365302466096dc0ea32926197f 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/7dab1cb040f0464894144c511165a0c9 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/132ce880b5a3430b859af884a6619d3d 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/a015b92f06ba41e8965c2e8815bc1881 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/a015b92f06ba41e8965c2e8815bc1881 2024-12-10T00:16:33,656 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/4720484840b242e383f20874c9b71939 2024-12-10T00:16:33,657 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/54b78bfc665c4ad382eed9021b2cea0b 2024-12-10T00:16:33,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bd9d9f84fd7d4c7d93819ab72ca97ac5, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/caaa882ead774747a9cedf5686538e94, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/47aa147d16ce436baab4108cb774294d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/662cbf48b77f48248aa06691c0f7da60, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/23e261253a70459591c139994e557950, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a091b1a17ec84f12a7ab263cda6402ae, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/e007a9fbcbe1470ebca4bbd2b9d0eeba, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a4c53ba8ff184f32aba357ccba7a9cc1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/5246e130bbe1411ca3e31867f1e4f0b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5] to archive 2024-12-10T00:16:33,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bdb28ca1f78a45c18a5cfac225430714 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bd9d9f84fd7d4c7d93819ab72ca97ac5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/bd9d9f84fd7d4c7d93819ab72ca97ac5 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b409edb35ce844d0a9b99994b260b7d6 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/caaa882ead774747a9cedf5686538e94 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/caaa882ead774747a9cedf5686538e94 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/3e555fb55ef04a089ced9b2219390f7a 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fca6df16d87c465580feda9fe1d51b59 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/b956e6631b004a89a8945aed5b5ec823 2024-12-10T00:16:33,661 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/834d194e6aa843488a3aed17498f8452 2024-12-10T00:16:33,662 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/64e37e4042464a2698bbacad1fda3676 2024-12-10T00:16:33,662 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/2041307ca9054fea9bf1c027ea398869 2024-12-10T00:16:33,662 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/47aa147d16ce436baab4108cb774294d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/47aa147d16ce436baab4108cb774294d 2024-12-10T00:16:33,662 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/70a9dbf5c0ef497fac4d29adad5860ea 2024-12-10T00:16:33,663 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/662cbf48b77f48248aa06691c0f7da60 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/662cbf48b77f48248aa06691c0f7da60 2024-12-10T00:16:33,663 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/643cdd283c1a443f8e7e099fd46b2651 2024-12-10T00:16:33,663 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/43e020997c3048968de677c0b5ea871a 2024-12-10T00:16:33,663 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/23e261253a70459591c139994e557950 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/23e261253a70459591c139994e557950 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/edbd94879a1f4c44a81d06c7d31ccb27 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/df699d53ab55497493e9c96f9aa1be36 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a091b1a17ec84f12a7ab263cda6402ae to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a091b1a17ec84f12a7ab263cda6402ae 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/e007a9fbcbe1470ebca4bbd2b9d0eeba to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/e007a9fbcbe1470ebca4bbd2b9d0eeba 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/7b559da1b7974673a285af02e08b5c6e 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/126913faa9ee4f59962c8f9503954235 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/39151fe9ff3942ad872fb2f3a345d3f2 2024-12-10T00:16:33,664 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/4aa754f687df44e9ac8a1d9a5855d9e3 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/beed95b834e949af9e2cae6012e84c85 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a4c53ba8ff184f32aba357ccba7a9cc1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/a4c53ba8ff184f32aba357ccba7a9cc1 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/5246e130bbe1411ca3e31867f1e4f0b6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/5246e130bbe1411ca3e31867f1e4f0b6 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/f3b0668d8dc243cf990ed776bb9d122c 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/fc14d5709bf04daaa01f48dac148595f 2024-12-10T00:16:33,665 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/9e19be0731234b31ae8a42de82e21584 2024-12-10T00:16:33,666 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/414e34a211b241c4b963380a1e01dbe5 2024-12-10T00:16:33,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/c06f70e19ab542b096ac8488a893f675, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/8872011ee09c4726ab07d1fdf12a941c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4e2f660f09684450ac922a52a6cd0724, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4faf7512055f467d8256ad1d0f539479, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fbd8979adf5145bc8350e328f2ba8663, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5f4e2410b8c44e2783574ed45e1f2871, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a0c4f669556349b3bdd062e3c77ccb29, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/730a1bc2c35543abb61d41d90024fe6b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ee77e4f79baa47d3884555d88a00f8f1, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf] to archive 2024-12-10T00:16:33,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ab25101d667b45b6b32eabde7e10814f 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0578dfc8ad1e415fb9381c01e511eac4 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fd4752a56243462d9208ca14d9157e28 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a15ea28a9d2c4074af3f3fc4eeebc339 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4dbea40934734a57be89eb4791d5b89c 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/cd9a5db5f891433fadb25aaae16ebb86 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/c06f70e19ab542b096ac8488a893f675 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/c06f70e19ab542b096ac8488a893f675 2024-12-10T00:16:33,669 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/8872011ee09c4726ab07d1fdf12a941c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/8872011ee09c4726ab07d1fdf12a941c 2024-12-10T00:16:33,670 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4e2f660f09684450ac922a52a6cd0724 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4e2f660f09684450ac922a52a6cd0724 2024-12-10T00:16:33,670 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/40a368b8f52f4ad1b59b7c26fed8507f 2024-12-10T00:16:33,670 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5b1e96d25fc44ffdb40c1ac94b8dd532 2024-12-10T00:16:33,671 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a96207b6548241619e836fc435d7412a 2024-12-10T00:16:33,671 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4faf7512055f467d8256ad1d0f539479 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/4faf7512055f467d8256ad1d0f539479 2024-12-10T00:16:33,671 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7bd2535f44d343f79a8bcea6f0ba7fa4 2024-12-10T00:16:33,671 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fbd8979adf5145bc8350e328f2ba8663 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/fbd8979adf5145bc8350e328f2ba8663 2024-12-10T00:16:33,671 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ec615b4903cb4efdb684fa4a6f13919a 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/3c38311edc9e476a96796e12db5b3eef 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/70e0a916976647c294fc1b2d3b86ace2 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0eea0647a70c4c44812144d7b865801f 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/6792aab36985465c8c3e5c4c651fb345 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5f4e2410b8c44e2783574ed45e1f2871 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/5f4e2410b8c44e2783574ed45e1f2871 2024-12-10T00:16:33,672 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a0c4f669556349b3bdd062e3c77ccb29 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a0c4f669556349b3bdd062e3c77ccb29 2024-12-10T00:16:33,673 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/a87fd85dbb494440828c1c30a6018b90 2024-12-10T00:16:33,673 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/0dfdcb98ceaa44aab9bb98dadadfffd9 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/b168bf7d62c2490ca107b09289894afd 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7959557c8a984d48b23f6413fce07970 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ee77e4f79baa47d3884555d88a00f8f1 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/ee77e4f79baa47d3884555d88a00f8f1 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/01992941690a411ba2fe7c1169afbd36 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/951c9d80b6c144be80aabbab635ae3c4 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/7b4d6ca648fd4af6a480a3524b10f0cf 2024-12-10T00:16:33,674 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/730a1bc2c35543abb61d41d90024fe6b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/730a1bc2c35543abb61d41d90024fe6b 2024-12-10T00:16:33,678 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=1 2024-12-10T00:16:33,679 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a. 2024-12-10T00:16:33,679 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for ee7312073a8c1deb0dc9c8e31f81137a: Waiting for close lock at 1733789792746Running coprocessor pre-close hooks at 1733789792746Disabling compacts and flushes for region at 1733789792746Disabling writes for close at 1733789793649 (+903 ms)Writing region close event to WAL at 1733789793675 (+26 ms)Running coprocessor post-close hooks at 1733789793679 (+4 ms)Closed at 1733789793679 2024-12-10T00:16:33,681 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:33,681 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ee7312073a8c1deb0dc9c8e31f81137a, regionState=CLOSED 2024-12-10T00:16:33,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:33,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-10T00:16:33,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure ee7312073a8c1deb0dc9c8e31f81137a, server=a67c4886b4f7,36911,1733789645811 in 1.0890 sec 2024-12-10T00:16:33,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-10T00:16:33,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee7312073a8c1deb0dc9c8e31f81137a, UNASSIGN in 1.0950 sec 2024-12-10T00:16:33,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-10T00:16:33,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0980 sec 2024-12-10T00:16:33,689 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789793689"}]},"ts":"1733789793689"} 2024-12-10T00:16:33,690 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:16:33,690 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:16:33,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1090 sec 2024-12-10T00:16:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-10T00:16:33,720 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:33,720 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,722 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-10T00:16:33,722 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,724 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:33,726 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/recovered.edits] 2024-12-10T00:16:33,729 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/9fa3a91de2ae4a16ba4fc4ac32e53bb3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/9fa3a91de2ae4a16ba4fc4ac32e53bb3 2024-12-10T00:16:33,729 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/0d0c7600122047fe9d7d6c15d677155e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/A/0d0c7600122047fe9d7d6c15d677155e 2024-12-10T00:16:33,730 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/c2e1c2ff59434716bc88d8f6c6a82eee to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/c2e1c2ff59434716bc88d8f6c6a82eee 2024-12-10T00:16:33,731 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/d1b0265444d342dcad5e76a235a16def to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/B/d1b0265444d342dcad5e76a235a16def 2024-12-10T00:16:33,732 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/277d02e44e994929ac832068771860d6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/277d02e44e994929ac832068771860d6 2024-12-10T00:16:33,732 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/89e6941a25214858bff59ee775570d7a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/C/89e6941a25214858bff59ee775570d7a 2024-12-10T00:16:33,734 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/recovered.edits/473.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a/recovered.edits/473.seqid 2024-12-10T00:16:33,734 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/ee7312073a8c1deb0dc9c8e31f81137a 2024-12-10T00:16:33,734 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:16:33,736 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,737 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:16:33,739 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:16:33,741 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,741 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:16:33,741 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789793741"}]},"ts":"9223372036854775807"} 2024-12-10T00:16:33,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:16:33,743 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee7312073a8c1deb0dc9c8e31f81137a, NAME => 'TestAcidGuarantees,,1733789769592.ee7312073a8c1deb0dc9c8e31f81137a.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:16:33,743 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:16:33,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789793743"}]},"ts":"9223372036854775807"} 2024-12-10T00:16:33,746 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:16:33,747 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-12-10T00:16:33,796 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:16:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-10T00:16:33,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:16:33,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:33,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=257 (was 255) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 465) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7065 (was 2915) - AvailableMemoryMB LEAK? - 2024-12-10T00:16:33,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=257, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=7062 2024-12-10T00:16:33,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:16:33,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:16:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:33,851 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T00:16:33,852 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:33,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-12-10T00:16:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:33,854 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T00:16:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742356_1532 (size=1061) 2024-12-10T00:16:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:34,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410 2024-12-10T00:16:34,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742357_1533 (size=53) 2024-12-10T00:16:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 62785890ba50211da2fffffcad482daf, disabling compactions & flushes 2024-12-10T00:16:34,673 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. after waiting 0 ms 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:34,673 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:34,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62785890ba50211da2fffffcad482daf: Waiting for close lock at 1733789794673Disabling compacts and flushes for region at 1733789794673Disabling writes for close at 1733789794673Writing region close event to WAL at 1733789794673Closed at 1733789794673 2024-12-10T00:16:34,675 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T00:16:34,675 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733789794675"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733789794675"}]},"ts":"1733789794675"} 2024-12-10T00:16:34,677 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T00:16:34,678 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T00:16:34,679 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789794678"}]},"ts":"1733789794678"} 2024-12-10T00:16:34,681 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T00:16:34,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, ASSIGN}] 2024-12-10T00:16:34,682 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, ASSIGN 2024-12-10T00:16:34,684 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, ASSIGN; state=OFFLINE, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=false 2024-12-10T00:16:34,835 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:34,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, ASSIGN because future has completed 2024-12-10T00:16:34,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:35,001 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:35,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:16:35,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:16:35,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,002 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,005 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,007 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:35,007 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName A 2024-12-10T00:16:35,007 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:35,008 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:35,009 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,011 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:35,011 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName B 2024-12-10T00:16:35,011 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:35,012 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:35,012 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,013 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:35,013 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName C 2024-12-10T00:16:35,013 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:35,014 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:35,014 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:35,014 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,015 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,015 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,016 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,016 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,017 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:16:35,018 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,020 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T00:16:35,020 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 62785890ba50211da2fffffcad482daf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62557900, jitterRate=-0.0678146481513977}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:16:35,020 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:35,021 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 62785890ba50211da2fffffcad482daf: Running coprocessor pre-open hook at 1733789795003Writing region info on filesystem at 1733789795003Initializing all the Stores at 1733789795004 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789795004Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789795004Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789795004Cleaning up temporary data from old regions at 1733789795016 (+12 ms)Running coprocessor post-open hooks at 1733789795020 (+4 ms)Region opened successfully at 1733789795021 (+1 ms) 2024-12-10T00:16:35,021 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., pid=148, masterSystemTime=1733789794995 2024-12-10T00:16:35,023 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:35,023 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:35,023 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=OPEN, openSeqNum=2, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:35,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:35,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-10T00:16:35,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 in 187 msec 2024-12-10T00:16:35,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-10T00:16:35,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, ASSIGN in 347 msec 2024-12-10T00:16:35,031 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T00:16:35,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789795031"}]},"ts":"1733789795031"} 2024-12-10T00:16:35,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T00:16:35,033 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T00:16:35,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-10T00:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T00:16:36,002 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:36,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:36,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:36,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:36,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9cddc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:36,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff28be2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:36,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:36,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:36,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T00:16:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a67c4886b4f7,35723,1733789644834 2024-12-10T00:16:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67cb4c53 2024-12-10T00:16:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T00:16:36,017 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T00:16:36,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T00:16:36,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T00:16:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T00:16:36,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742358_1534 (size=1097) 2024-12-10T00:16:36,432 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T00:16:36,432 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T00:16:36,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:16:36,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE}] 2024-12-10T00:16:36,441 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE 2024-12-10T00:16:36,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:36,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE because future has completed 2024-12-10T00:16:36,443 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:16:36,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:36,596 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:36,596 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:16:36,596 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 62785890ba50211da2fffffcad482daf, disabling compactions & flushes 2024-12-10T00:16:36,596 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:36,596 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:36,597 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. after waiting 0 ms 2024-12-10T00:16:36,597 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:36,788 INFO [AsyncFSWAL-0-hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410-prefix:a67c4886b4f7,36911,1733789645811 {}] wal.AbstractFSWAL(1368): Slow sync cost: 190 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33323,DS-f7201ec0-4054-4520-8b26-bc2011faafef,DISK]] 2024-12-10T00:16:36,790 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T00:16:36,790 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:36,791 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 62785890ba50211da2fffffcad482daf: Waiting for close lock at 1733789796596Running coprocessor pre-close hooks at 1733789796596Disabling compacts and flushes for region at 1733789796596Disabling writes for close at 1733789796597 (+1 ms)Writing region close event to WAL at 1733789796597Running coprocessor post-close hooks at 1733789796790 (+193 ms)Closed at 1733789796790 2024-12-10T00:16:36,791 WARN [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 62785890ba50211da2fffffcad482daf to self. 2024-12-10T00:16:36,792 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:36,793 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=CLOSED 2024-12-10T00:16:36,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:36,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-10T00:16:36,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 in 351 msec 2024-12-10T00:16:36,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE; state=CLOSED, location=a67c4886b4f7,36911,1733789645811; forceNewPlan=false, retain=true 2024-12-10T00:16:36,947 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=OPENING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:36,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE because future has completed 2024-12-10T00:16:36,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:16:37,110 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,111 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} 2024-12-10T00:16:37,111 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,111 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T00:16:37,111 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,111 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,112 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,113 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:37,113 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName A 2024-12-10T00:16:37,114 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:37,114 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:37,114 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,115 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:37,115 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName B 2024-12-10T00:16:37,115 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:37,115 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:37,115 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,116 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T00:16:37,116 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62785890ba50211da2fffffcad482daf columnFamilyName C 2024-12-10T00:16:37,116 DEBUG [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:37,116 INFO [StoreOpener-62785890ba50211da2fffffcad482daf-1 {}] regionserver.HStore(327): Store=62785890ba50211da2fffffcad482daf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T00:16:37,116 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,116 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,117 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,118 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,119 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,119 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,119 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T00:16:37,120 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,121 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 62785890ba50211da2fffffcad482daf; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61727775, jitterRate=-0.08018447458744049}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T00:16:37,121 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,121 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 62785890ba50211da2fffffcad482daf: Running coprocessor pre-open hook at 1733789797111Writing region info on filesystem at 1733789797111Initializing all the Stores at 1733789797112 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789797112Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789797112Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733789797112Cleaning up temporary data from old regions at 1733789797119 (+7 ms)Running coprocessor post-open hooks at 1733789797121 (+2 ms)Region opened successfully at 1733789797121 2024-12-10T00:16:37,122 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., pid=153, masterSystemTime=1733789797107 2024-12-10T00:16:37,124 DEBUG [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,124 INFO [RS_OPEN_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=OPEN, openSeqNum=5, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:16:37,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-12-10T00:16:37,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 in 174 msec 2024-12-10T00:16:37,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-10T00:16:37,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, REOPEN/MOVE in 687 msec 2024-12-10T00:16:37,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-10T00:16:37,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 691 msec 2024-12-10T00:16:37,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1130 sec 2024-12-10T00:16:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-10T00:16:37,132 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:37,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,134 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae9b448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,135 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d36c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,137 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@901b327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,138 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c68ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,139 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac13d80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,140 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2895ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a473ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc36bf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,145 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f15d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,146 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b66b386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694353fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f865ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2517816d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42995279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f6e39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bc56f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b81e43c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,153 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41353664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a2ffd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,155 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fd5a47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,157 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51eb4c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fba46bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67c4886b4f7,35723,-1 for getting cluster id 2024-12-10T00:16:37,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T00:16:37,158 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a958df3-9645-4748-962c-d558219e47f8' 2024-12-10T00:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T00:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a958df3-9645-4748-962c-d558219e47f8" 2024-12-10T00:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42465441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67c4886b4f7,35723,-1] 2024-12-10T00:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T00:16:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:37,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T00:16:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140dd81b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T00:16:37,160 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,160 DEBUG [Thread-2387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,160 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,161 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,161 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,161 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,162 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,162 DEBUG [Thread-2381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,163 DEBUG [Thread-2396 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,163 DEBUG [Thread-2396 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,165 DEBUG [Thread-2383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-10T00:16:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:37,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:37,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:37,181 DEBUG [Thread-2385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T00:16:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67c4886b4f7,36911,1733789645811, seqNum=-1] 2024-12-10T00:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T00:16:37,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T00:16:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5] 2024-12-10T00:16:37,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121042681967a7274121ab750198d3318c05_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789797171/Put/seqid=0 2024-12-10T00:16:37,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789857197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789857197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789797190, max timestamp=1733789797199Num uniques 859; 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789797190, max timestamp=1733789797199Num uniques 859; 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789797190, max timestamp=1733789797199Num uniques 859; 2024-12-10T00:16:37,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:37,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742359_1535 (size=17034) 2024-12-10T00:16:37,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789857218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789857219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789857221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789857309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789857309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789857330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789857330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789857330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:37,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789857519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789857519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789857540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789857540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789857540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:37,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121042681967a7274121ab750198d3318c05_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042681967a7274121ab750198d3318c05_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:37,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/23003b4e90bc45cc80a9237e0c762ab6, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:37,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/23003b4e90bc45cc80a9237e0c762ab6 is 175, key is test_row_0/A:col10/1733789797171/Put/seqid=0 2024-12-10T00:16:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742360_1536 (size=48139) 2024-12-10T00:16:37,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:37,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789857829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789857829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789857850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789857850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789857850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:37,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:37,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:37,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:38,012 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/23003b4e90bc45cc80a9237e0c762ab6 2024-12-10T00:16:38,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/c1a1281c49964acf938ca1a511cb45dd is 50, key is test_row_0/B:col10/1733789797171/Put/seqid=0 2024-12-10T00:16:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742361_1537 (size=12001) 2024-12-10T00:16:38,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/c1a1281c49964acf938ca1a511cb45dd 2024-12-10T00:16:38,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/88766c9c284946cbbfafc5bb9340c989 is 50, key is test_row_0/C:col10/1733789797171/Put/seqid=0 2024-12-10T00:16:38,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742362_1538 (size=12001) 2024-12-10T00:16:38,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/88766c9c284946cbbfafc5bb9340c989 2024-12-10T00:16:38,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/23003b4e90bc45cc80a9237e0c762ab6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6 2024-12-10T00:16:38,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6, entries=250, sequenceid=15, filesize=47.0 K 2024-12-10T00:16:38,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/c1a1281c49964acf938ca1a511cb45dd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd 2024-12-10T00:16:38,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T00:16:38,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/88766c9c284946cbbfafc5bb9340c989 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989 2024-12-10T00:16:38,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:38,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:38,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989, entries=150, sequenceid=15, filesize=11.7 K 2024-12-10T00:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:38,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 62785890ba50211da2fffffcad482daf in 914ms, sequenceid=15, compaction requested=false 2024-12-10T00:16:38,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:38,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-10T00:16:38,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:38,239 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121033641bdde1b042289eca153b0059b743_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789797215/Put/seqid=0 2024-12-10T00:16:38,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742363_1539 (size=12154) 2024-12-10T00:16:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:38,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789858355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789858357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789858359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789858359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789858359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789858469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789858469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:38,651 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121033641bdde1b042289eca153b0059b743_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121033641bdde1b042289eca153b0059b743_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:38,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e60a9877c3534cd3936a8507028b68d2, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:38,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e60a9877c3534cd3936a8507028b68d2 is 175, key is test_row_0/A:col10/1733789797215/Put/seqid=0 2024-12-10T00:16:38,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742364_1540 (size=30955) 2024-12-10T00:16:38,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789858679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789858679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789858990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789858990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,056 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e60a9877c3534cd3936a8507028b68d2 2024-12-10T00:16:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/025aa954973b4d0b8c3942636ae2f9f9 is 50, key is test_row_0/B:col10/1733789797215/Put/seqid=0 2024-12-10T00:16:39,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742365_1541 (size=12001) 2024-12-10T00:16:39,198 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T00:16:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:39,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789859369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:39,372 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789859380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:39,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789859380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:39,383 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:39,383 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,464 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/025aa954973b4d0b8c3942636ae2f9f9 2024-12-10T00:16:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4bf61c18d084b5e97b8e4c175fa8e6f is 50, key is test_row_0/C:col10/1733789797215/Put/seqid=0 2024-12-10T00:16:39,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742366_1542 (size=12001) 2024-12-10T00:16:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789859500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789859500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:39,875 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4bf61c18d084b5e97b8e4c175fa8e6f 2024-12-10T00:16:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e60a9877c3534cd3936a8507028b68d2 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2 2024-12-10T00:16:39,882 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2, entries=150, sequenceid=51, filesize=30.2 K 2024-12-10T00:16:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/025aa954973b4d0b8c3942636ae2f9f9 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9 2024-12-10T00:16:39,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T00:16:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4bf61c18d084b5e97b8e4c175fa8e6f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f 2024-12-10T00:16:39,889 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T00:16:39,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 62785890ba50211da2fffffcad482daf in 1651ms, sequenceid=51, compaction requested=false 2024-12-10T00:16:39,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:39,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:39,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-10T00:16:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-10T00:16:39,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-10T00:16:39,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7200 sec 2024-12-10T00:16:39,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7270 sec 2024-12-10T00:16:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:40,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:40,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:40,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f27a5c6b10a34e3ead41fa374cfcddf1_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:40,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742367_1543 (size=17034) 2024-12-10T00:16:40,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789860547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789860550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789860660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789860660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789860869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789860869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:40,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:40,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:40,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f27a5c6b10a34e3ead41fa374cfcddf1_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27a5c6b10a34e3ead41fa374cfcddf1_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:40,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/ee4887d7c7a44a87a398ec48acb37eeb, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:40,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/ee4887d7c7a44a87a398ec48acb37eeb is 175, key is test_row_0/A:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:40,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742368_1544 (size=48139) 2024-12-10T00:16:40,945 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/ee4887d7c7a44a87a398ec48acb37eeb 2024-12-10T00:16:40,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/009c49f47e634b7c8324ac3745a6d965 is 50, key is test_row_0/B:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:40,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742369_1545 (size=12001) 2024-12-10T00:16:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789861180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789861180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-10T00:16:41,339 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:41,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-10T00:16:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:41,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:41,342 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:41,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789861380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,383 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/009c49f47e634b7c8324ac3745a6d965 2024-12-10T00:16:41,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/854177dddf77413ba4a572495b7f44fc is 50, key is test_row_0/C:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:41,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742370_1546 (size=12001) 2024-12-10T00:16:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789861409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789861409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,413 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,413 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:41,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-10T00:16:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-10T00:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:41,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789861689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789861689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:41,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-10T00:16:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:41,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/854177dddf77413ba4a572495b7f44fc 2024-12-10T00:16:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/ee4887d7c7a44a87a398ec48acb37eeb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb 2024-12-10T00:16:41,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb, entries=250, sequenceid=64, filesize=47.0 K 2024-12-10T00:16:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/009c49f47e634b7c8324ac3745a6d965 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965 2024-12-10T00:16:41,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965, entries=150, sequenceid=64, filesize=11.7 K 2024-12-10T00:16:41,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/854177dddf77413ba4a572495b7f44fc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc 2024-12-10T00:16:41,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc, entries=150, sequenceid=64, filesize=11.7 K 2024-12-10T00:16:41,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 62785890ba50211da2fffffcad482daf in 1303ms, sequenceid=64, compaction requested=true 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:41,816 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:41,816 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:41,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:41,817 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,817 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.2 K 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:41,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=124.3 K 2024-12-10T00:16:41,817 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb] 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a1281c49964acf938ca1a511cb45dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789797169 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23003b4e90bc45cc80a9237e0c762ab6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789797169 2024-12-10T00:16:41,817 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 025aa954973b4d0b8c3942636ae2f9f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733789797215 2024-12-10T00:16:41,818 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60a9877c3534cd3936a8507028b68d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733789797215 2024-12-10T00:16:41,818 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 009c49f47e634b7c8324ac3745a6d965, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798354 2024-12-10T00:16:41,818 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee4887d7c7a44a87a398ec48acb37eeb, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798348 2024-12-10T00:16:41,837 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:41,838 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:41,839 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/57ed8359a07645c885224d95b26b0d87 is 50, key is test_row_0/B:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:41,841 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210c80fd2fdee084933821d9d83edd265a3_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:41,843 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210c80fd2fdee084933821d9d83edd265a3_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:41,843 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c80fd2fdee084933821d9d83edd265a3_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:41,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742371_1547 (size=12104) 2024-12-10T00:16:41,861 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/57ed8359a07645c885224d95b26b0d87 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/57ed8359a07645c885224d95b26b0d87 2024-12-10T00:16:41,865 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into 57ed8359a07645c885224d95b26b0d87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:41,866 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:41,866 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=13, startTime=1733789801816; duration=0sec 2024-12-10T00:16:41,866 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:41,866 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:41,866 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:41,867 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:41,867 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:41,867 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,867 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.2 K 2024-12-10T00:16:41,867 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 88766c9c284946cbbfafc5bb9340c989, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733789797169 2024-12-10T00:16:41,867 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bf61c18d084b5e97b8e4c175fa8e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733789797215 2024-12-10T00:16:41,868 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 854177dddf77413ba4a572495b7f44fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798354 2024-12-10T00:16:41,875 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:41,875 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/9a397a6f14ac4c6da34304d0764fc7a4 is 50, key is test_row_0/C:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:41,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742372_1548 (size=4469) 2024-12-10T00:16:41,883 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#470 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:41,884 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/c395de1e1e184123abe9f28866d82eb8 is 175, key is test_row_0/A:col10/1733789798355/Put/seqid=0 2024-12-10T00:16:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742373_1549 (size=12104) 2024-12-10T00:16:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742374_1550 (size=31058) 2024-12-10T00:16:41,892 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/c395de1e1e184123abe9f28866d82eb8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8 2024-12-10T00:16:41,896 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into c395de1e1e184123abe9f28866d82eb8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:41,896 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:41,896 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=13, startTime=1733789801816; duration=0sec 2024-12-10T00:16:41,896 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:41,896 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:41,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-10T00:16:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:41,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:41,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100a49374adbf44ca0ab82c287fa72836d_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789800533/Put/seqid=0 2024-12-10T00:16:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742375_1551 (size=12154) 2024-12-10T00:16:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:42,285 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/9a397a6f14ac4c6da34304d0764fc7a4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/9a397a6f14ac4c6da34304d0764fc7a4 2024-12-10T00:16:42,289 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into 9a397a6f14ac4c6da34304d0764fc7a4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:42,289 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:42,289 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=13, startTime=1733789801816; duration=0sec 2024-12-10T00:16:42,289 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:42,289 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:42,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:42,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100a49374adbf44ca0ab82c287fa72836d_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100a49374adbf44ca0ab82c287fa72836d_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/209b22f6357446069ab6295d970d73db, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/209b22f6357446069ab6295d970d73db is 175, key is test_row_0/A:col10/1733789800533/Put/seqid=0 2024-12-10T00:16:42,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742376_1552 (size=30955) 2024-12-10T00:16:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:42,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:42,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789862724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789862726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:42,772 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/209b22f6357446069ab6295d970d73db 2024-12-10T00:16:42,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/150664b0fe1b46469c0950f87d00b7db is 50, key is test_row_0/B:col10/1733789800533/Put/seqid=0 2024-12-10T00:16:42,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742377_1553 (size=12001) 2024-12-10T00:16:42,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/150664b0fe1b46469c0950f87d00b7db 2024-12-10T00:16:42,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/cb50850106c948669fd10423f37518ae is 50, key is test_row_0/C:col10/1733789800533/Put/seqid=0 2024-12-10T00:16:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742378_1554 (size=12001) 2024-12-10T00:16:42,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789862830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:42,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789862830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789863039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789863039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,201 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/cb50850106c948669fd10423f37518ae 2024-12-10T00:16:43,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/209b22f6357446069ab6295d970d73db as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db 2024-12-10T00:16:43,207 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db, entries=150, sequenceid=85, filesize=30.2 K 2024-12-10T00:16:43,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/150664b0fe1b46469c0950f87d00b7db as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db 2024-12-10T00:16:43,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db, entries=150, sequenceid=85, filesize=11.7 K 2024-12-10T00:16:43,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/cb50850106c948669fd10423f37518ae as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae 2024-12-10T00:16:43,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae, entries=150, sequenceid=85, filesize=11.7 K 2024-12-10T00:16:43,220 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 62785890ba50211da2fffffcad482daf in 1267ms, sequenceid=85, compaction requested=false 2024-12-10T00:16:43,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:43,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-10T00:16:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-10T00:16:43,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-10T00:16:43,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-12-10T00:16:43,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8830 sec 2024-12-10T00:16:43,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:43,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:43,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101dec37e89f0e4369ab28bc1476bd0ecd_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:43,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742379_1555 (size=17034) 2024-12-10T00:16:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789863386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789863386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-10T00:16:43,499 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:43,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:43,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789863499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789863499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:43,505 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:43,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:43,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:43,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789863709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789863709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:43,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:43,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101dec37e89f0e4369ab28bc1476bd0ecd_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101dec37e89f0e4369ab28bc1476bd0ecd_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:43,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/601228095fdb46eaa835131c50fde86f, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:43,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/601228095fdb46eaa835131c50fde86f is 175, key is test_row_0/A:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:43,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742380_1556 (size=48139) 2024-12-10T00:16:43,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:43,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:43,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789864019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:44,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789864020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:44,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:44,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:44,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:44,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/601228095fdb46eaa835131c50fde86f 2024-12-10T00:16:44,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0986da5a98ea42f69525c145098cde6f is 50, key is test_row_0/B:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742381_1557 (size=12001) 2024-12-10T00:16:44,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:44,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:44,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789864529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:44,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789864529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:44,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:44,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0986da5a98ea42f69525c145098cde6f 2024-12-10T00:16:44,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d9ed62149f8a48eda05c5e7fcd11b57a is 50, key is test_row_0/C:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:44,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742382_1558 (size=12001) 2024-12-10T00:16:44,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d9ed62149f8a48eda05c5e7fcd11b57a 2024-12-10T00:16:44,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/601228095fdb46eaa835131c50fde86f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f 2024-12-10T00:16:44,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f, entries=250, sequenceid=101, filesize=47.0 K 2024-12-10T00:16:44,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0986da5a98ea42f69525c145098cde6f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f 2024-12-10T00:16:44,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f, entries=150, sequenceid=101, filesize=11.7 K 2024-12-10T00:16:44,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d9ed62149f8a48eda05c5e7fcd11b57a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a 2024-12-10T00:16:44,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a, entries=150, sequenceid=101, filesize=11.7 K 2024-12-10T00:16:44,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 62785890ba50211da2fffffcad482daf in 1263ms, sequenceid=101, compaction requested=true 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:44,615 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:44,616 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,616 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,616 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/57ed8359a07645c885224d95b26b0d87, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.3 K 2024-12-10T00:16:44,616 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=107.6 K 2024-12-10T00:16:44,616 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f] 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ed8359a07645c885224d95b26b0d87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798354 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c395de1e1e184123abe9f28866d82eb8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798354 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 150664b0fe1b46469c0950f87d00b7db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733789800532 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209b22f6357446069ab6295d970d73db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733789800532 2024-12-10T00:16:44,616 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0986da5a98ea42f69525c145098cde6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802720 2024-12-10T00:16:44,617 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 601228095fdb46eaa835131c50fde86f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802718 2024-12-10T00:16:44,622 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:44,623 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412103c536a79b6b84c4083bd2b6e3def1c4e_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:44,624 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:44,624 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/847c4310335d4332ab9f32f64a612d18 is 50, key is test_row_0/B:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:44,625 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412103c536a79b6b84c4083bd2b6e3def1c4e_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:44,625 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103c536a79b6b84c4083bd2b6e3def1c4e_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:44,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742383_1559 (size=12207) 2024-12-10T00:16:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742384_1560 (size=4469) 2024-12-10T00:16:44,632 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/847c4310335d4332ab9f32f64a612d18 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/847c4310335d4332ab9f32f64a612d18 2024-12-10T00:16:44,637 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into 847c4310335d4332ab9f32f64a612d18(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:44,637 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:44,637 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=13, startTime=1733789804615; duration=0sec 2024-12-10T00:16:44,637 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:44,637 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:44,637 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:44,638 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:44,638 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:44,638 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,638 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/9a397a6f14ac4c6da34304d0764fc7a4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.3 K 2024-12-10T00:16:44,638 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a397a6f14ac4c6da34304d0764fc7a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733789798354 2024-12-10T00:16:44,638 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cb50850106c948669fd10423f37518ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733789800532 2024-12-10T00:16:44,639 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d9ed62149f8a48eda05c5e7fcd11b57a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802720 2024-12-10T00:16:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:44,646 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:44,646 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/908c475345ba4daeb2c3934d36b6e5b6 is 50, key is test_row_0/C:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:44,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742385_1561 (size=12207) 2024-12-10T00:16:44,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:44,725 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:44,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:44,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fc4495307cab4e97a5d48f13a5494c6a_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789803384/Put/seqid=0 2024-12-10T00:16:44,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742386_1562 (size=12154) 2024-12-10T00:16:45,033 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:45,033 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/270fc66891264b3eb10616492ac57b1b is 175, key is test_row_0/A:col10/1733789803350/Put/seqid=0 2024-12-10T00:16:45,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742387_1563 (size=31161) 2024-12-10T00:16:45,056 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/908c475345ba4daeb2c3934d36b6e5b6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/908c475345ba4daeb2c3934d36b6e5b6 2024-12-10T00:16:45,059 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into 908c475345ba4daeb2c3934d36b6e5b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:45,059 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:45,059 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=13, startTime=1733789804615; duration=0sec 2024-12-10T00:16:45,059 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:45,059 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:45,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:45,136 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fc4495307cab4e97a5d48f13a5494c6a_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc4495307cab4e97a5d48f13a5494c6a_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/30fa531f9c3745b797ad35cddf3827f7, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/30fa531f9c3745b797ad35cddf3827f7 is 175, key is test_row_0/A:col10/1733789803384/Put/seqid=0 2024-12-10T00:16:45,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742388_1564 (size=30955) 2024-12-10T00:16:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:45,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:45,440 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/270fc66891264b3eb10616492ac57b1b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b 2024-12-10T00:16:45,446 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into 270fc66891264b3eb10616492ac57b1b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:45,446 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:45,446 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=13, startTime=1733789804615; duration=0sec 2024-12-10T00:16:45,446 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:45,446 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:45,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789865450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789865450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,452 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:45,452 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789865452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,540 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/30fa531f9c3745b797ad35cddf3827f7 2024-12-10T00:16:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789865540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,540 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/ebbb0aa841d241e6871bd4bbf0e7687c is 50, key is test_row_0/B:col10/1733789803384/Put/seqid=0 2024-12-10T00:16:45,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789865550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742389_1565 (size=12001) 2024-12-10T00:16:45,553 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789865559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789865770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:45,954 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/ebbb0aa841d241e6871bd4bbf0e7687c 2024-12-10T00:16:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/fd03b1b489264a03803c06b082fb7b40 is 50, key is test_row_0/C:col10/1733789803384/Put/seqid=0 2024-12-10T00:16:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742390_1566 (size=12001) 2024-12-10T00:16:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789866079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:46,363 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/fd03b1b489264a03803c06b082fb7b40 2024-12-10T00:16:46,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/30fa531f9c3745b797ad35cddf3827f7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7 2024-12-10T00:16:46,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7, entries=150, sequenceid=120, filesize=30.2 K 2024-12-10T00:16:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/ebbb0aa841d241e6871bd4bbf0e7687c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c 2024-12-10T00:16:46,425 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T00:16:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/fd03b1b489264a03803c06b082fb7b40 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40 2024-12-10T00:16:46,428 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T00:16:46,428 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 62785890ba50211da2fffffcad482daf in 1703ms, sequenceid=120, compaction requested=false 2024-12-10T00:16:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-10T00:16:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-10T00:16:46,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-10T00:16:46,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-12-10T00:16:46,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.9320 sec 2024-12-10T00:16:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:46,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T00:16:46,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:46,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:46,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cab6fa4705244404bf04156277f872b0_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:46,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742391_1567 (size=12254) 2024-12-10T00:16:46,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:46,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cab6fa4705244404bf04156277f872b0_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cab6fa4705244404bf04156277f872b0_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:46,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4e08cf944879425dae175ef90786e0bb, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:46,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4e08cf944879425dae175ef90786e0bb is 175, key is test_row_0/A:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:46,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742392_1568 (size=31055) 2024-12-10T00:16:46,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4e08cf944879425dae175ef90786e0bb 2024-12-10T00:16:46,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0189d962dbad40bf9fab985e098c7e08 is 50, key is test_row_0/B:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:46,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742393_1569 (size=12101) 2024-12-10T00:16:46,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0189d962dbad40bf9fab985e098c7e08 2024-12-10T00:16:46,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/54e111d9cfa04c87849b254edaf8bc1e is 50, key is test_row_0/C:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:46,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742394_1570 (size=12101) 2024-12-10T00:16:46,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789866648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:46,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789866759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789866969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/54e111d9cfa04c87849b254edaf8bc1e 2024-12-10T00:16:47,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4e08cf944879425dae175ef90786e0bb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb 2024-12-10T00:16:47,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb, entries=150, sequenceid=137, filesize=30.3 K 2024-12-10T00:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/0189d962dbad40bf9fab985e098c7e08 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08 2024-12-10T00:16:47,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08, entries=150, sequenceid=137, filesize=11.8 K 2024-12-10T00:16:47,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/54e111d9cfa04c87849b254edaf8bc1e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e 2024-12-10T00:16:47,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e, entries=150, sequenceid=137, filesize=11.8 K 2024-12-10T00:16:47,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 62785890ba50211da2fffffcad482daf in 464ms, sequenceid=137, compaction requested=true 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:47,055 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:47,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:47,055 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:47,056 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,056 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,056 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/847c4310335d4332ab9f32f64a612d18, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.5 K 2024-12-10T00:16:47,056 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=91.0 K 2024-12-10T00:16:47,056 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb] 2024-12-10T00:16:47,056 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 847c4310335d4332ab9f32f64a612d18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802720 2024-12-10T00:16:47,057 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270fc66891264b3eb10616492ac57b1b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802720 2024-12-10T00:16:47,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ebbb0aa841d241e6871bd4bbf0e7687c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733789803379 2024-12-10T00:16:47,057 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30fa531f9c3745b797ad35cddf3827f7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733789803379 2024-12-10T00:16:47,057 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0189d962dbad40bf9fab985e098c7e08, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:47,057 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e08cf944879425dae175ef90786e0bb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:47,065 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:47,066 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121096ef2490e507492e8fa882d961f1552f_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:47,067 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:47,067 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/a856a21c04fb44abb928186f24638efc is 50, key is test_row_0/B:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:47,068 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121096ef2490e507492e8fa882d961f1552f_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:47,068 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121096ef2490e507492e8fa882d961f1552f_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742395_1571 (size=12409) 2024-12-10T00:16:47,073 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/a856a21c04fb44abb928186f24638efc as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/a856a21c04fb44abb928186f24638efc 2024-12-10T00:16:47,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742396_1572 (size=4469) 2024-12-10T00:16:47,077 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into a856a21c04fb44abb928186f24638efc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:47,077 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:47,077 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=13, startTime=1733789807055; duration=0sec 2024-12-10T00:16:47,077 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:47,077 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:47,077 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:47,078 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:47,078 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:47,078 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,078 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/908c475345ba4daeb2c3934d36b6e5b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=35.5 K 2024-12-10T00:16:47,079 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 908c475345ba4daeb2c3934d36b6e5b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733789802720 2024-12-10T00:16:47,080 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fd03b1b489264a03803c06b082fb7b40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733789803379 2024-12-10T00:16:47,080 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e111d9cfa04c87849b254edaf8bc1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:47,084 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:47,085 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/53d78377a55d4a24b08811bd816b3a17 is 50, key is test_row_0/C:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742397_1573 (size=12409) 2024-12-10T00:16:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:47,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:47,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:47,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e1ed472bff154b4aa8bac08ddb59027d_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789806600/Put/seqid=0 2024-12-10T00:16:47,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742398_1574 (size=14794) 2024-12-10T00:16:47,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789867319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789867429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,475 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:47,476 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/1d091b6046904a6aae6d2d12432c413a is 175, key is test_row_0/A:col10/1733789805420/Put/seqid=0 2024-12-10T00:16:47,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742399_1575 (size=31363) 2024-12-10T00:16:47,484 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/1d091b6046904a6aae6d2d12432c413a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a 2024-12-10T00:16:47,487 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into 1d091b6046904a6aae6d2d12432c413a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:47,487 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:47,487 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=13, startTime=1733789807055; duration=0sec 2024-12-10T00:16:47,487 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:47,487 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:47,491 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/53d78377a55d4a24b08811bd816b3a17 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/53d78377a55d4a24b08811bd816b3a17 2024-12-10T00:16:47,494 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into 53d78377a55d4a24b08811bd816b3a17(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:47,494 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:47,494 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=13, startTime=1733789807055; duration=0sec 2024-12-10T00:16:47,494 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:47,494 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:47,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789867559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,561 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789867569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,571 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789867640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-10T00:16:47,659 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:47,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-10T00:16:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:47,661 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:47,662 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:47,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:47,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:47,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e1ed472bff154b4aa8bac08ddb59027d_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1ed472bff154b4aa8bac08ddb59027d_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:47,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:47,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e is 175, key is test_row_0/A:col10/1733789806600/Put/seqid=0 2024-12-10T00:16:47,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742400_1576 (size=39749) 2024-12-10T00:16:47,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e 2024-12-10T00:16:47,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/fa0d4fb7e13849aeb151eeb2554703e3 is 50, key is test_row_0/B:col10/1733789806600/Put/seqid=0 2024-12-10T00:16:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742401_1577 (size=12151) 2024-12-10T00:16:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:47,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789867949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:47,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:47,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:48,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/fa0d4fb7e13849aeb151eeb2554703e3 2024-12-10T00:16:48,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/56b4f8f414d9412ab90f911c858eb7b6 is 50, key is test_row_0/C:col10/1733789806600/Put/seqid=0 2024-12-10T00:16:48,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742402_1578 (size=12151) 2024-12-10T00:16:48,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:48,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:48,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:48,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789868459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:48,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/56b4f8f414d9412ab90f911c858eb7b6 2024-12-10T00:16:48,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e 2024-12-10T00:16:48,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e, entries=200, sequenceid=156, filesize=38.8 K 2024-12-10T00:16:48,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/fa0d4fb7e13849aeb151eeb2554703e3 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3 2024-12-10T00:16:48,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T00:16:48,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/56b4f8f414d9412ab90f911c858eb7b6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6 2024-12-10T00:16:48,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T00:16:48,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 62785890ba50211da2fffffcad482daf in 1252ms, sequenceid=156, compaction requested=false 2024-12-10T00:16:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:48,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-10T00:16:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:48,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:48,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210065808890dd54482aca47bbf4691fb47_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789807318/Put/seqid=0 2024-12-10T00:16:48,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742403_1579 (size=12304) 2024-12-10T00:16:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:48,588 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210065808890dd54482aca47bbf4691fb47_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210065808890dd54482aca47bbf4691fb47_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/429e6c895f5e41fa870f01ae411f09cf, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:48,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/429e6c895f5e41fa870f01ae411f09cf is 175, key is test_row_0/A:col10/1733789807318/Put/seqid=0 2024-12-10T00:16:48,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742404_1580 (size=31105) 2024-12-10T00:16:48,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/429e6c895f5e41fa870f01ae411f09cf 2024-12-10T00:16:48,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/6eef520cc54c4bc1b80ff5736e209dce is 50, key is test_row_0/B:col10/1733789807318/Put/seqid=0 2024-12-10T00:16:48,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742405_1581 (size=12151) 2024-12-10T00:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:49,003 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/6eef520cc54c4bc1b80ff5736e209dce 2024-12-10T00:16:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/7a96a9a6a93c415ebc100e79194e1b1d is 50, key is test_row_0/C:col10/1733789807318/Put/seqid=0 2024-12-10T00:16:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742406_1582 (size=12151) 2024-12-10T00:16:49,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/7a96a9a6a93c415ebc100e79194e1b1d 2024-12-10T00:16:49,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/429e6c895f5e41fa870f01ae411f09cf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf 2024-12-10T00:16:49,425 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf, entries=150, sequenceid=172, filesize=30.4 K 2024-12-10T00:16:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/6eef520cc54c4bc1b80ff5736e209dce as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce 2024-12-10T00:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,428 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce, entries=150, sequenceid=172, filesize=11.9 K 2024-12-10T00:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/7a96a9a6a93c415ebc100e79194e1b1d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d 2024-12-10T00:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d, entries=150, sequenceid=172, filesize=11.9 K 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 62785890ba50211da2fffffcad482daf in 858ms, sequenceid=172, compaction requested=true 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-10T00:16:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-10T00:16:49,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7730 sec 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7770 sec 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:49,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:49,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:49,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108639052ec5f14150897858083e12e9c2_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:49,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742407_1583 (size=14794) 2024-12-10T00:16:49,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:49,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108639052ec5f14150897858083e12e9c2_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108639052ec5f14150897858083e12e9c2_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:49,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/5deb7f48ba4a4010b210960add1dc9f7, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:49,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/5deb7f48ba4a4010b210960add1dc9f7 is 175, key is test_row_0/A:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:49,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742408_1584 (size=39749) 2024-12-10T00:16:49,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789809575, max timestamp=1733789809594Num uniques 859; 2024-12-10T00:16:49,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:49,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789809575, max timestamp=1733789809594Num uniques 859; 2024-12-10T00:16:49,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:49,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789809575, max timestamp=1733789809594Num uniques 859; 2024-12-10T00:16:49,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:49,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789869668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:49,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789869779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-10T00:16:49,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:49,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-10T00:16:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:49,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:49,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:49,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:49,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/5deb7f48ba4a4010b210960add1dc9f7 2024-12-10T00:16:49,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/d9147fc43cda45659ba218d370159251 is 50, key is test_row_0/B:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:49,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742409_1585 (size=12151) 2024-12-10T00:16:49,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-10T00:16:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:49,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:49,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789869990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:50,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-10T00:16:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:50,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-10T00:16:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789870299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:50,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/d9147fc43cda45659ba218d370159251 2024-12-10T00:16:50,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4369ce874354604a65745594da82020 is 50, key is test_row_0/C:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:50,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742410_1586 (size=12151) 2024-12-10T00:16:50,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4369ce874354604a65745594da82020 2024-12-10T00:16:50,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/5deb7f48ba4a4010b210960add1dc9f7 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7 2024-12-10T00:16:50,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7, entries=200, sequenceid=182, filesize=38.8 K 2024-12-10T00:16:50,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/d9147fc43cda45659ba218d370159251 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251 2024-12-10T00:16:50,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T00:16:50,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/d4369ce874354604a65745594da82020 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020 2024-12-10T00:16:50,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T00:16:50,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 62785890ba50211da2fffffcad482daf in 862ms, sequenceid=182, compaction requested=true 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:50,387 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:50,387 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:50,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:50,388 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:50,388 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:50,388 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:50,388 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:50,388 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,388 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,389 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=138.6 K 2024-12-10T00:16:50,389 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/a856a21c04fb44abb928186f24638efc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=47.7 K 2024-12-10T00:16:50,389 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7] 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a856a21c04fb44abb928186f24638efc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d091b6046904a6aae6d2d12432c413a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fa0d4fb7e13849aeb151eeb2554703e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733789806600 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3eae4243db4a5d9e9f6b1ab08cbf3e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733789806600 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eef520cc54c4bc1b80ff5736e209dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733789807304 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429e6c895f5e41fa870f01ae411f09cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733789807304 2024-12-10T00:16:50,389 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5deb7f48ba4a4010b210960add1dc9f7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:50,390 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d9147fc43cda45659ba218d370159251, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:50,394 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:50,395 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:50,395 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/967a1515f0a742fd8119386c4aa3931b is 50, key is test_row_0/B:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:50,396 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210ce7bede503ff4125b218d0135a88641e_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:50,398 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210ce7bede503ff4125b218d0135a88641e_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:50,398 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ce7bede503ff4125b218d0135a88641e_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:50,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742411_1587 (size=12595) 2024-12-10T00:16:50,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742412_1588 (size=4469) 2024-12-10T00:16:50,413 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/967a1515f0a742fd8119386c4aa3931b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/967a1515f0a742fd8119386c4aa3931b 2024-12-10T00:16:50,419 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into 967a1515f0a742fd8119386c4aa3931b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:50,419 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:50,419 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=12, startTime=1733789810387; duration=0sec 2024-12-10T00:16:50,419 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:50,419 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:50,419 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:50,420 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:50,420 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:50,420 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,420 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/53d78377a55d4a24b08811bd816b3a17, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=47.7 K 2024-12-10T00:16:50,421 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d78377a55d4a24b08811bd816b3a17, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733789805420 2024-12-10T00:16:50,421 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b4f8f414d9412ab90f911c858eb7b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733789806600 2024-12-10T00:16:50,421 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a96a9a6a93c415ebc100e79194e1b1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733789807304 2024-12-10T00:16:50,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-10T00:16:50,422 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d4369ce874354604a65745594da82020, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:50,422 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:50,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:50,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108b72e22fbce94db4a3f52d56a5443788_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:50,429 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:50,429 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/ebd508c871574bbcae622a789df6f2c4 is 50, key is test_row_0/C:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:50,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742413_1589 (size=12304) 2024-12-10T00:16:50,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:50,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742414_1590 (size=12595) 2024-12-10T00:16:50,471 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108b72e22fbce94db4a3f52d56a5443788_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b72e22fbce94db4a3f52d56a5443788_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:50,471 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/ebd508c871574bbcae622a789df6f2c4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/ebd508c871574bbcae622a789df6f2c4 2024-12-10T00:16:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/3a1ca8bf454e4b5aaf15a24523a5427f, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/3a1ca8bf454e4b5aaf15a24523a5427f is 175, key is test_row_0/A:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:50,476 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into ebd508c871574bbcae622a789df6f2c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:50,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:50,477 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=12, startTime=1733789810387; duration=0sec 2024-12-10T00:16:50,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:50,477 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:50,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742415_1591 (size=31105) 2024-12-10T00:16:50,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/3a1ca8bf454e4b5aaf15a24523a5427f 2024-12-10T00:16:50,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/509f43490f9541acb108b83920f12c48 is 50, key is test_row_0/B:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:50,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742416_1592 (size=12151) 2024-12-10T00:16:50,810 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#500 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:50,811 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4a60131ba29e449d8c5df5a1e4afe43e is 175, key is test_row_0/A:col10/1733789809520/Put/seqid=0 2024-12-10T00:16:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:50,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742417_1593 (size=31549) 2024-12-10T00:16:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789870838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:50,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/509f43490f9541acb108b83920f12c48 2024-12-10T00:16:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/12106c68b088484fa0d4b7504cee392a is 50, key is test_row_0/C:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742418_1594 (size=12151) 2024-12-10T00:16:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:50,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789870950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789871159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,217 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/4a60131ba29e449d8c5df5a1e4afe43e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e 2024-12-10T00:16:51,220 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into 4a60131ba29e449d8c5df5a1e4afe43e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:51,220 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:51,220 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=12, startTime=1733789810387; duration=0sec 2024-12-10T00:16:51,220 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:51,220 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/12106c68b088484fa0d4b7504cee392a 2024-12-10T00:16:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/3a1ca8bf454e4b5aaf15a24523a5427f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f 2024-12-10T00:16:51,311 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f, entries=150, sequenceid=220, filesize=30.4 K 2024-12-10T00:16:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/509f43490f9541acb108b83920f12c48 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48 2024-12-10T00:16:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,315 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48, entries=150, sequenceid=220, filesize=11.9 K 2024-12-10T00:16:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/12106c68b088484fa0d4b7504cee392a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a 2024-12-10T00:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,318 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a, entries=150, sequenceid=220, filesize=11.9 K 2024-12-10T00:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,322 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 62785890ba50211da2fffffcad482daf in 900ms, sequenceid=220, compaction requested=false 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-10T00:16:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-10T00:16:51,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-12-10T00:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5160 sec 2024-12-10T00:16:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:51,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121093bdb407021e47fda88fec65c3f23306_62785890ba50211da2fffffcad482daf is 50, key is test_row_1/A:col10/1733789811474/Put/seqid=0 2024-12-10T00:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742419_1595 (size=12304) 2024-12-10T00:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,487 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,490 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121093bdb407021e47fda88fec65c3f23306_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093bdb407021e47fda88fec65c3f23306_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/dc25b0410f8540fd8388e91d322a89a8, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/dc25b0410f8540fd8388e91d322a89a8 is 175, key is test_row_1/A:col10/1733789811474/Put/seqid=0 2024-12-10T00:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742420_1596 (size=31101) 2024-12-10T00:16:51,517 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/dc25b0410f8540fd8388e91d322a89a8 2024-12-10T00:16:51,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/431e2f3a9a0a45548758c7ad224f94eb is 50, key is test_row_1/B:col10/1733789811474/Put/seqid=0 2024-12-10T00:16:51,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742421_1597 (size=9757) 2024-12-10T00:16:51,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/431e2f3a9a0a45548758c7ad224f94eb 2024-12-10T00:16:51,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/2606116184e74a52a4aa5debae31320a is 50, key is test_row_1/C:col10/1733789811474/Put/seqid=0 2024-12-10T00:16:51,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742422_1598 (size=9757) 2024-12-10T00:16:51,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/2606116184e74a52a4aa5debae31320a 2024-12-10T00:16:51,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/dc25b0410f8540fd8388e91d322a89a8 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8 2024-12-10T00:16:51,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8, entries=150, sequenceid=233, filesize=30.4 K 2024-12-10T00:16:51,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/431e2f3a9a0a45548758c7ad224f94eb as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb 2024-12-10T00:16:51,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb, entries=100, sequenceid=233, filesize=9.5 K 2024-12-10T00:16:51,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/2606116184e74a52a4aa5debae31320a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a 2024-12-10T00:16:51,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a, entries=100, sequenceid=233, filesize=9.5 K 2024-12-10T00:16:51,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=114.05 KB/116790 for 62785890ba50211da2fffffcad482daf in 89ms, sequenceid=233, compaction requested=true 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:51,564 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:51,564 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811527, max timestamp=1733789811565Num uniques 859; 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811527, max timestamp=1733789811565Num uniques 859; 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811527, max timestamp=1733789811565Num uniques 859; 2024-12-10T00:16:51,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:51,568 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:51,568 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=91.6 K 2024-12-10T00:16:51,568 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8] 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:51,568 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a60131ba29e449d8c5df5a1e4afe43e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:51,568 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/967a1515f0a742fd8119386c4aa3931b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=33.7 K 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1ca8bf454e4b5aaf15a24523a5427f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733789809618 2024-12-10T00:16:51,568 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 967a1515f0a742fd8119386c4aa3931b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:51,569 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 509f43490f9541acb108b83920f12c48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733789809618 2024-12-10T00:16:51,569 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc25b0410f8540fd8388e91d322a89a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789810820 2024-12-10T00:16:51,569 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 431e2f3a9a0a45548758c7ad224f94eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789810837 2024-12-10T00:16:51,576 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,578 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:51,578 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/36e019b6be9745b094d3e5ed93b482dd is 50, key is test_row_0/B:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:51,580 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412105fe3113489af48c180e2c60b415b9d4a_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:51,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:51,582 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412105fe3113489af48c180e2c60b415b9d4a_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:51,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:51,582 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105fe3113489af48c180e2c60b415b9d4a_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742423_1599 (size=12697) 2024-12-10T00:16:51,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742424_1600 (size=4469) 2024-12-10T00:16:51,594 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#508 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:51,594 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/401cf8d1691d4e85b98d2d90a3ec4c3d is 175, key is test_row_0/A:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:51,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121093990f367a224ff8a681a344ed69c416_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789811575/Put/seqid=0 2024-12-10T00:16:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742425_1601 (size=31758) 2024-12-10T00:16:51,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742426_1602 (size=14894) 2024-12-10T00:16:51,600 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:51,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121093990f367a224ff8a681a344ed69c416_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093990f367a224ff8a681a344ed69c416_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:51,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/8a4cc5f9235845efbe9e8ed30f1dae8c, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:51,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/8a4cc5f9235845efbe9e8ed30f1dae8c is 175, key is test_row_0/A:col10/1733789811575/Put/seqid=0 2024-12-10T00:16:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742427_1603 (size=39849) 2024-12-10T00:16:51,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811613, max timestamp=1733789811622Num uniques 859; 2024-12-10T00:16:51,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811613, max timestamp=1733789811622Num uniques 859; 2024-12-10T00:16:51,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733789811613, max timestamp=1733789811622Num uniques 859; 2024-12-10T00:16:51,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T00:16:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789871645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789871646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789871649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789871760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789871760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789871760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-10T00:16:51,960 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:51,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-10T00:16:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:51,961 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:51,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:51,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789871970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789871970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789871970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:51,992 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/36e019b6be9745b094d3e5ed93b482dd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/36e019b6be9745b094d3e5ed93b482dd 2024-12-10T00:16:51,996 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into 36e019b6be9745b094d3e5ed93b482dd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:51,996 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:51,996 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=13, startTime=1733789811564; duration=0sec 2024-12-10T00:16:51,996 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:51,996 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:51,996 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T00:16:51,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T00:16:51,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:51,997 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:51,997 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/ebd508c871574bbcae622a789df6f2c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=33.7 K 2024-12-10T00:16:51,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd508c871574bbcae622a789df6f2c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733789809512 2024-12-10T00:16:51,997 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 12106c68b088484fa0d4b7504cee392a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733789809618 2024-12-10T00:16:51,998 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2606116184e74a52a4aa5debae31320a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789810837 2024-12-10T00:16:52,003 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:52,004 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/261def4a64df4daabcd07272d7dcda59 is 50, key is test_row_0/C:col10/1733789809667/Put/seqid=0 2024-12-10T00:16:52,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/8a4cc5f9235845efbe9e8ed30f1dae8c 2024-12-10T00:16:52,010 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/401cf8d1691d4e85b98d2d90a3ec4c3d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d 2024-12-10T00:16:52,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742428_1604 (size=12697) 2024-12-10T00:16:52,019 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into 401cf8d1691d4e85b98d2d90a3ec4c3d(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:52,019 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:52,019 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=13, startTime=1733789811564; duration=0sec 2024-12-10T00:16:52,019 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:52,019 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:52,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/b0bca637d4e94ec0833eca4e23990d3a is 50, key is test_row_0/B:col10/1733789811575/Put/seqid=0 2024-12-10T00:16:52,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742429_1605 (size=12251) 2024-12-10T00:16:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:52,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:52,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789872279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789872279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789872280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,421 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/261def4a64df4daabcd07272d7dcda59 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/261def4a64df4daabcd07272d7dcda59 2024-12-10T00:16:52,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/b0bca637d4e94ec0833eca4e23990d3a 2024-12-10T00:16:52,425 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into 261def4a64df4daabcd07272d7dcda59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:52,425 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:52,425 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=13, startTime=1733789811564; duration=0sec 2024-12-10T00:16:52,425 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:52,425 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:52,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/8769c3cf13d041beb5c058867b5a108f is 50, key is test_row_0/C:col10/1733789811575/Put/seqid=0 2024-12-10T00:16:52,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742430_1606 (size=12251) 2024-12-10T00:16:52,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:52,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:52,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:52,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789872789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789872790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789872790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:52,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/8769c3cf13d041beb5c058867b5a108f 2024-12-10T00:16:52,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/8a4cc5f9235845efbe9e8ed30f1dae8c as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c 2024-12-10T00:16:52,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c, entries=200, sequenceid=258, filesize=38.9 K 2024-12-10T00:16:52,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/b0bca637d4e94ec0833eca4e23990d3a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a 2024-12-10T00:16:52,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T00:16:52,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/8769c3cf13d041beb5c058867b5a108f as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f 2024-12-10T00:16:52,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T00:16:52,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 62785890ba50211da2fffffcad482daf in 1269ms, sequenceid=258, compaction requested=false 2024-12-10T00:16:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:52,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:52,876 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:52,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105e0985a53b734cec8716504359aa9243_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789811638/Put/seqid=0 2024-12-10T00:16:52,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742431_1607 (size=12454) 2024-12-10T00:16:52,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:52,889 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105e0985a53b734cec8716504359aa9243_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e0985a53b734cec8716504359aa9243_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/bdc634c9976e41c6a4d3ffd80a7b0b36, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/bdc634c9976e41c6a4d3ffd80a7b0b36 is 175, key is test_row_0/A:col10/1733789811638/Put/seqid=0 2024-12-10T00:16:52,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742432_1608 (size=31255) 2024-12-10T00:16:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:53,293 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/bdc634c9976e41c6a4d3ffd80a7b0b36 2024-12-10T00:16:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/de252abab1cd49439e49220b022398a0 is 50, key is test_row_0/B:col10/1733789811638/Put/seqid=0 2024-12-10T00:16:53,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742433_1609 (size=12301) 2024-12-10T00:16:53,701 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/de252abab1cd49439e49220b022398a0 2024-12-10T00:16:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/bf7f1df343e6432ebb9195e2d703c723 is 50, key is test_row_0/C:col10/1733789811638/Put/seqid=0 2024-12-10T00:16:53,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742434_1610 (size=12301) 2024-12-10T00:16:53,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789873814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:53,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789873815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:53,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789873817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789873919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:53,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789873930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789873930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:54,110 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/bf7f1df343e6432ebb9195e2d703c723 2024-12-10T00:16:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/bdc634c9976e41c6a4d3ffd80a7b0b36 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36 2024-12-10T00:16:54,115 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36, entries=150, sequenceid=298, filesize=30.5 K 2024-12-10T00:16:54,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/de252abab1cd49439e49220b022398a0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0 2024-12-10T00:16:54,118 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0, entries=150, sequenceid=298, filesize=12.0 K 2024-12-10T00:16:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/bf7f1df343e6432ebb9195e2d703c723 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723 2024-12-10T00:16:54,122 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723, entries=150, sequenceid=298, filesize=12.0 K 2024-12-10T00:16:54,123 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 62785890ba50211da2fffffcad482daf in 1246ms, sequenceid=298, compaction requested=true 2024-12-10T00:16:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-10T00:16:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-10T00:16:54,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-10T00:16:54,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-12-10T00:16:54,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.1660 sec 2024-12-10T00:16:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:54,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:54,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103d7c0696c88444118ed50c1626d1f286_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:54,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742435_1611 (size=14994) 2024-12-10T00:16:54,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789814153, max timestamp=1733789814165Num uniques 859; 2024-12-10T00:16:54,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:54,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789814153, max timestamp=1733789814165Num uniques 859; 2024-12-10T00:16:54,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:54,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:54,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733789814153, max timestamp=1733789814165Num uniques 859; 2024-12-10T00:16:54,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a67c4886b4f7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T00:16:54,169 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103d7c0696c88444118ed50c1626d1f286_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d7c0696c88444118ed50c1626d1f286_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:54,170 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/95307ae79e80448e9f391e55a1969387, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:54,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/95307ae79e80448e9f391e55a1969387 is 175, key is test_row_0/A:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:54,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742436_1612 (size=39949) 2024-12-10T00:16:54,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/95307ae79e80448e9f391e55a1969387 2024-12-10T00:16:54,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/8e8af838d0a143038ae6c5c56ce9a4cf is 50, key is test_row_0/B:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789874190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789874191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742437_1613 (size=12301) 2024-12-10T00:16:54,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789874191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789874300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789874300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789874300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789874509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789874509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789874510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/8e8af838d0a143038ae6c5c56ce9a4cf 2024-12-10T00:16:54,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/98d9a1b6d1e24926ac57da58e8d7a16b is 50, key is test_row_0/C:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742438_1614 (size=12301) 2024-12-10T00:16:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789874819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789874819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789874819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:54,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/98d9a1b6d1e24926ac57da58e8d7a16b 2024-12-10T00:16:55,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/95307ae79e80448e9f391e55a1969387 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387 2024-12-10T00:16:55,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387, entries=200, sequenceid=309, filesize=39.0 K 2024-12-10T00:16:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/8e8af838d0a143038ae6c5c56ce9a4cf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf 2024-12-10T00:16:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T00:16:55,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/98d9a1b6d1e24926ac57da58e8d7a16b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b 2024-12-10T00:16:55,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T00:16:55,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 62785890ba50211da2fffffcad482daf in 883ms, sequenceid=309, compaction requested=true 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T00:16:55,015 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62785890ba50211da2fffffcad482daf:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T00:16:55,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:55,015 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:55,016 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:55,016 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142811 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:55,016 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/B is initiating minor compaction (all files) 2024-12-10T00:16:55,016 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/A is initiating minor compaction (all files) 2024-12-10T00:16:55,016 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/A in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:55,016 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/B in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:55,016 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/36e019b6be9745b094d3e5ed93b482dd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=48.4 K 2024-12-10T00:16:55,016 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=139.5 K 2024-12-10T00:16:55,016 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:55,016 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. files: [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387] 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e019b6be9745b094d3e5ed93b482dd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789809667 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 401cf8d1691d4e85b98d2d90a3ec4c3d, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789809667 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bca637d4e94ec0833eca4e23990d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733789811560 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4cc5f9235845efbe9e8ed30f1dae8c, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733789811560 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdc634c9976e41c6a4d3ffd80a7b0b36, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733789811638 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting de252abab1cd49439e49220b022398a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733789811638 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95307ae79e80448e9f391e55a1969387, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789813800 2024-12-10T00:16:55,017 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8af838d0a143038ae6c5c56ce9a4cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789813800 2024-12-10T00:16:55,026 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:55,027 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:55,028 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/ddd1282f9ef34ba99f2f4dcc6b648518 is 50, key is test_row_0/B:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:55,028 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210cee4db2ebace4ae1925d03ed471c7d80_62785890ba50211da2fffffcad482daf store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:55,030 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210cee4db2ebace4ae1925d03ed471c7d80_62785890ba50211da2fffffcad482daf, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:55,030 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210cee4db2ebace4ae1925d03ed471c7d80_62785890ba50211da2fffffcad482daf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:55,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742439_1615 (size=12983) 2024-12-10T00:16:55,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742440_1616 (size=4469) 2024-12-10T00:16:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:55,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T00:16:55,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108de05a9b784645aa9f8e029ed3bb0cec_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789814185/Put/seqid=0 2024-12-10T00:16:55,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742441_1617 (size=14994) 2024-12-10T00:16:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789875338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789875342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789875343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,434 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#A#compaction#521 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:55,434 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e30dee084312439a85795217b78b2823 is 175, key is test_row_0/A:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:55,436 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/ddd1282f9ef34ba99f2f4dcc6b648518 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ddd1282f9ef34ba99f2f4dcc6b648518 2024-12-10T00:16:55,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742442_1618 (size=31937) 2024-12-10T00:16:55,444 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/B of 62785890ba50211da2fffffcad482daf into ddd1282f9ef34ba99f2f4dcc6b648518(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:55,444 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:55,444 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/B, priority=12, startTime=1733789815015; duration=0sec 2024-12-10T00:16:55,444 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T00:16:55,444 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:B 2024-12-10T00:16:55,444 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T00:16:55,445 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/e30dee084312439a85795217b78b2823 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e30dee084312439a85795217b78b2823 2024-12-10T00:16:55,445 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T00:16:55,445 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1541): 62785890ba50211da2fffffcad482daf/C is initiating minor compaction (all files) 2024-12-10T00:16:55,445 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62785890ba50211da2fffffcad482daf/C in TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:55,446 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/261def4a64df4daabcd07272d7dcda59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp, totalSize=48.4 K 2024-12-10T00:16:55,446 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 261def4a64df4daabcd07272d7dcda59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733789809667 2024-12-10T00:16:55,446 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8769c3cf13d041beb5c058867b5a108f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733789811560 2024-12-10T00:16:55,446 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bf7f1df343e6432ebb9195e2d703c723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733789811638 2024-12-10T00:16:55,447 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d9a1b6d1e24926ac57da58e8d7a16b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733789813800 2024-12-10T00:16:55,449 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/A of 62785890ba50211da2fffffcad482daf into e30dee084312439a85795217b78b2823(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:55,449 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:55,450 INFO [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/A, priority=12, startTime=1733789815015; duration=0sec 2024-12-10T00:16:55,450 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:55,450 DEBUG [RS:0;a67c4886b4f7:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:A 2024-12-10T00:16:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789875450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789875450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789875450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,461 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62785890ba50211da2fffffcad482daf#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T00:16:55,462 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/3de020e9c30247c8afb2461b5dbf9099 is 50, key is test_row_0/C:col10/1733789813816/Put/seqid=0 2024-12-10T00:16:55,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742443_1619 (size=12983) 2024-12-10T00:16:55,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47432 deadline: 1733789875479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,482 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_0' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1733789875550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,552 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a67c4886b4f7,36911,1733789645811 for 'test_row_2' in 62785890ba50211da2fffffcad482daf of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18333 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T00:16:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789875660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789875660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789875660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:55,744 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108de05a9b784645aa9f8e029ed3bb0cec_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108de05a9b784645aa9f8e029ed3bb0cec_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:55,745 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/66aace7b465f4b02bee790424148e2f0, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:55,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/66aace7b465f4b02bee790424148e2f0 is 175, key is test_row_0/A:col10/1733789814185/Put/seqid=0 2024-12-10T00:16:55,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742444_1620 (size=39949) 2024-12-10T00:16:55,873 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/3de020e9c30247c8afb2461b5dbf9099 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/3de020e9c30247c8afb2461b5dbf9099 2024-12-10T00:16:55,876 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62785890ba50211da2fffffcad482daf/C of 62785890ba50211da2fffffcad482daf into 3de020e9c30247c8afb2461b5dbf9099(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T00:16:55,876 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:55,876 INFO [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., storeName=62785890ba50211da2fffffcad482daf/C, priority=12, startTime=1733789815015; duration=0sec 2024-12-10T00:16:55,876 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T00:16:55,876 DEBUG [RS:0;a67c4886b4f7:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62785890ba50211da2fffffcad482daf:C 2024-12-10T00:16:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789875969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789875969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789875970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-10T00:16:56,130 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:16:56,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-10T00:16:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-10T00:16:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:56,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T00:16:56,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T00:16:56,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T00:16:56,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/66aace7b465f4b02bee790424148e2f0 2024-12-10T00:16:56,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/10b6732701b24193a31945bd04032104 is 50, key is test_row_0/B:col10/1733789814185/Put/seqid=0 2024-12-10T00:16:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742445_1621 (size=12301) 2024-12-10T00:16:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:56,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:56,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47362 deadline: 1733789876480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47422 deadline: 1733789876480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47398 deadline: 1733789876480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 , the old value is region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T00:16:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., hostname=a67c4886b4f7,36911,1733789645811, seqNum=5 because the exception is null or not the one we care about 2024-12-10T00:16:56,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/10b6732701b24193a31945bd04032104 2024-12-10T00:16:56,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a04badc64ca342d0bfce9aedfce65798 is 50, key is test_row_0/C:col10/1733789814185/Put/seqid=0 2024-12-10T00:16:56,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742446_1622 (size=12301) 2024-12-10T00:16:56,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:56,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:56,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T00:16:56,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a04badc64ca342d0bfce9aedfce65798 2024-12-10T00:16:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/66aace7b465f4b02bee790424148e2f0 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/66aace7b465f4b02bee790424148e2f0 2024-12-10T00:16:56,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/66aace7b465f4b02bee790424148e2f0, entries=200, sequenceid=344, filesize=39.0 K 2024-12-10T00:16:56,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/10b6732701b24193a31945bd04032104 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/10b6732701b24193a31945bd04032104 2024-12-10T00:16:56,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/10b6732701b24193a31945bd04032104, entries=150, sequenceid=344, filesize=12.0 K 2024-12-10T00:16:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a04badc64ca342d0bfce9aedfce65798 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a04badc64ca342d0bfce9aedfce65798 2024-12-10T00:16:56,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a04badc64ca342d0bfce9aedfce65798, entries=150, sequenceid=344, filesize=12.0 K 2024-12-10T00:16:56,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 62785890ba50211da2fffffcad482daf in 1648ms, sequenceid=344, compaction requested=false 2024-12-10T00:16:56,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:57,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-10T00:16:57,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:57,047 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:16:57,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121016ddd1e7570d4c8099f197305e908f10_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789815341/Put/seqid=0 2024-12-10T00:16:57,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742447_1623 (size=12454) 2024-12-10T00:16:57,177 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-12-10T00:16:57,177 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,177 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,177 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,177 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-12-10T00:16:57,177 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,177 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,178 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,179 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-12-10T00:16:57,179 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,179 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,179 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,179 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-12-10T00:16:57,179 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,179 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,179 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,179 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-12-10T00:16:57,179 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,179 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,179 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:57,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:16:57,460 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121016ddd1e7570d4c8099f197305e908f10_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016ddd1e7570d4c8099f197305e908f10_62785890ba50211da2fffffcad482daf 2024-12-10T00:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/234ff83cab604596a43e278391b2706d, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/234ff83cab604596a43e278391b2706d is 175, key is test_row_0/A:col10/1733789815341/Put/seqid=0 2024-12-10T00:16:57,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742448_1624 (size=31255) 2024-12-10T00:16:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 62785890ba50211da2fffffcad482daf 2024-12-10T00:16:57,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. as already flushing 2024-12-10T00:16:57,493 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-12-10T00:16:57,493 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,493 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,493 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,494 INFO [Thread-2381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2381. 2024-12-10T00:16:57,494 INFO [Thread-2383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2383. 2024-12-10T00:16:57,494 DEBUG [Thread-2381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,494 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,494 DEBUG [Thread-2383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:16:57,494 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,494 DEBUG [Thread-2381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,494 DEBUG [Thread-2383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:16:57,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:16:57,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/234ff83cab604596a43e278391b2706d 2024-12-10T00:16:57,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/3d0f0a0806e8407bbb1722e4fe7fa032 is 50, key is test_row_0/B:col10/1733789815341/Put/seqid=0 2024-12-10T00:16:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742449_1625 (size=12301) 2024-12-10T00:16:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:16:58,285 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/3d0f0a0806e8407bbb1722e4fe7fa032 2024-12-10T00:16:58,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/007acb6390dc46e38200d8eed1051f9b is 50, key is test_row_0/C:col10/1733789815341/Put/seqid=0 2024-12-10T00:16:58,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742450_1626 (size=12301) 2024-12-10T00:16:58,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/007acb6390dc46e38200d8eed1051f9b 2024-12-10T00:16:58,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/234ff83cab604596a43e278391b2706d as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/234ff83cab604596a43e278391b2706d 2024-12-10T00:16:58,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/234ff83cab604596a43e278391b2706d, entries=150, sequenceid=357, filesize=30.5 K 2024-12-10T00:16:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/3d0f0a0806e8407bbb1722e4fe7fa032 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/3d0f0a0806e8407bbb1722e4fe7fa032 2024-12-10T00:16:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/3d0f0a0806e8407bbb1722e4fe7fa032, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T00:16:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/007acb6390dc46e38200d8eed1051f9b as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/007acb6390dc46e38200d8eed1051f9b 2024-12-10T00:16:58,729 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/007acb6390dc46e38200d8eed1051f9b, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T00:16:58,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 62785890ba50211da2fffffcad482daf in 1684ms, sequenceid=357, compaction requested=true 2024-12-10T00:16:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 62785890ba50211da2fffffcad482daf: 2024-12-10T00:16:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:16:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a67c4886b4f7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-10T00:16:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-10T00:16:58,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-10T00:16:58,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5990 sec 2024-12-10T00:16:58,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.6050 sec 2024-12-10T00:17:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-10T00:17:00,302 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:17:03,796 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T00:17:05,564 INFO [Thread-2385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2385. 2024-12-10T00:17:05,564 DEBUG [Thread-2385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:17:05,564 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,564 DEBUG [Thread-2385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:17:05,614 INFO [Thread-2387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2387. 2024-12-10T00:17:05,614 DEBUG [Thread-2387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T00:17:05,614 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,614 DEBUG [Thread-2387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4431 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4432 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4309 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4434 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4419 2024-12-10T00:17:05,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T00:17:05,616 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:17:05,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:17:05,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:05,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:17:05,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-10T00:17:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:05,623 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789825622"}]},"ts":"1733789825622"} 2024-12-10T00:17:05,624 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T00:17:05,625 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T00:17:05,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T00:17:05,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, UNASSIGN}] 2024-12-10T00:17:05,628 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, UNASSIGN 2024-12-10T00:17:05,629 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=CLOSING, regionLocation=a67c4886b4f7,36911,1733789645811 2024-12-10T00:17:05,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, UNASSIGN because future has completed 2024-12-10T00:17:05,630 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T00:17:05,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811}] 2024-12-10T00:17:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:05,783 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 62785890ba50211da2fffffcad482daf 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 62785890ba50211da2fffffcad482daf, disabling compactions & flushes 2024-12-10T00:17:05,783 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. after waiting 0 ms 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:17:05,783 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 62785890ba50211da2fffffcad482daf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=A 2024-12-10T00:17:05,783 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:17:05,784 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=B 2024-12-10T00:17:05,784 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:17:05,784 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 62785890ba50211da2fffffcad482daf, store=C 2024-12-10T00:17:05,784 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T00:17:05,792 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101495f9ec078340c3a182bfed01d98ab3_62785890ba50211da2fffffcad482daf is 50, key is test_row_0/A:col10/1733789825561/Put/seqid=0 2024-12-10T00:17:05,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742451_1627 (size=12454) 2024-12-10T00:17:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:06,197 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T00:17:06,205 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101495f9ec078340c3a182bfed01d98ab3_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101495f9ec078340c3a182bfed01d98ab3_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:06,206 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/b5025343934442acb0db2706d4bbb096, store: [table=TestAcidGuarantees family=A region=62785890ba50211da2fffffcad482daf] 2024-12-10T00:17:06,207 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/b5025343934442acb0db2706d4bbb096 is 175, key is test_row_0/A:col10/1733789825561/Put/seqid=0 2024-12-10T00:17:06,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742452_1628 (size=31255) 2024-12-10T00:17:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:06,612 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/b5025343934442acb0db2706d4bbb096 2024-12-10T00:17:06,623 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/abb3fc01d638461cab1b18211601f5e6 is 50, key is test_row_0/B:col10/1733789825561/Put/seqid=0 2024-12-10T00:17:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742453_1629 (size=12301) 2024-12-10T00:17:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:07,028 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/abb3fc01d638461cab1b18211601f5e6 2024-12-10T00:17:07,040 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a2ebdfddd0b041bc985e020c3b0f2dcf is 50, key is test_row_0/C:col10/1733789825561/Put/seqid=0 2024-12-10T00:17:07,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742454_1630 (size=12301) 2024-12-10T00:17:07,044 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a2ebdfddd0b041bc985e020c3b0f2dcf 2024-12-10T00:17:07,049 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/A/b5025343934442acb0db2706d4bbb096 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/b5025343934442acb0db2706d4bbb096 2024-12-10T00:17:07,054 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/b5025343934442acb0db2706d4bbb096, entries=150, sequenceid=365, filesize=30.5 K 2024-12-10T00:17:07,055 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/B/abb3fc01d638461cab1b18211601f5e6 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/abb3fc01d638461cab1b18211601f5e6 2024-12-10T00:17:07,059 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/abb3fc01d638461cab1b18211601f5e6, entries=150, sequenceid=365, filesize=12.0 K 2024-12-10T00:17:07,060 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/.tmp/C/a2ebdfddd0b041bc985e020c3b0f2dcf as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a2ebdfddd0b041bc985e020c3b0f2dcf 2024-12-10T00:17:07,064 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a2ebdfddd0b041bc985e020c3b0f2dcf, entries=150, sequenceid=365, filesize=12.0 K 2024-12-10T00:17:07,065 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 62785890ba50211da2fffffcad482daf in 1282ms, sequenceid=365, compaction requested=true 2024-12-10T00:17:07,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387] to archive 2024-12-10T00:17:07,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:17:07,068 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/23003b4e90bc45cc80a9237e0c762ab6 2024-12-10T00:17:07,068 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e60a9877c3534cd3936a8507028b68d2 2024-12-10T00:17:07,068 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/ee4887d7c7a44a87a398ec48acb37eeb 2024-12-10T00:17:07,068 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/c395de1e1e184123abe9f28866d82eb8 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/209b22f6357446069ab6295d970d73db 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/270fc66891264b3eb10616492ac57b1b 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/601228095fdb46eaa835131c50fde86f 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/30fa531f9c3745b797ad35cddf3827f7 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/1d091b6046904a6aae6d2d12432c413a 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4e08cf944879425dae175ef90786e0bb 2024-12-10T00:17:07,069 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/429e6c895f5e41fa870f01ae411f09cf 2024-12-10T00:17:07,070 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/fb3eae4243db4a5d9e9f6b1ab08cbf3e 2024-12-10T00:17:07,070 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/5deb7f48ba4a4010b210960add1dc9f7 2024-12-10T00:17:07,070 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/4a60131ba29e449d8c5df5a1e4afe43e 2024-12-10T00:17:07,070 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/401cf8d1691d4e85b98d2d90a3ec4c3d 2024-12-10T00:17:07,070 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/3a1ca8bf454e4b5aaf15a24523a5427f 2024-12-10T00:17:07,071 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/bdc634c9976e41c6a4d3ffd80a7b0b36 2024-12-10T00:17:07,071 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/95307ae79e80448e9f391e55a1969387 2024-12-10T00:17:07,073 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/dc25b0410f8540fd8388e91d322a89a8 2024-12-10T00:17:07,073 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/8a4cc5f9235845efbe9e8ed30f1dae8c 2024-12-10T00:17:07,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/57ed8359a07645c885224d95b26b0d87, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/847c4310335d4332ab9f32f64a612d18, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/a856a21c04fb44abb928186f24638efc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/967a1515f0a742fd8119386c4aa3931b, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/36e019b6be9745b094d3e5ed93b482dd, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf] to archive 2024-12-10T00:17:07,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/c1a1281c49964acf938ca1a511cb45dd 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/025aa954973b4d0b8c3942636ae2f9f9 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/57ed8359a07645c885224d95b26b0d87 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/57ed8359a07645c885224d95b26b0d87 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/847c4310335d4332ab9f32f64a612d18 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/847c4310335d4332ab9f32f64a612d18 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0986da5a98ea42f69525c145098cde6f 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ebbb0aa841d241e6871bd4bbf0e7687c 2024-12-10T00:17:07,076 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/150664b0fe1b46469c0950f87d00b7db 2024-12-10T00:17:07,077 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/a856a21c04fb44abb928186f24638efc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/a856a21c04fb44abb928186f24638efc 2024-12-10T00:17:07,077 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/6eef520cc54c4bc1b80ff5736e209dce 2024-12-10T00:17:07,077 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/967a1515f0a742fd8119386c4aa3931b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/967a1515f0a742fd8119386c4aa3931b 2024-12-10T00:17:07,077 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/fa0d4fb7e13849aeb151eeb2554703e3 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/d9147fc43cda45659ba218d370159251 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/509f43490f9541acb108b83920f12c48 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/0189d962dbad40bf9fab985e098c7e08 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/36e019b6be9745b094d3e5ed93b482dd to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/36e019b6be9745b094d3e5ed93b482dd 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/b0bca637d4e94ec0833eca4e23990d3a 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/431e2f3a9a0a45548758c7ad224f94eb 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/009c49f47e634b7c8324ac3745a6d965 2024-12-10T00:17:07,078 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/8e8af838d0a143038ae6c5c56ce9a4cf 2024-12-10T00:17:07,079 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/de252abab1cd49439e49220b022398a0 2024-12-10T00:17:07,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/9a397a6f14ac4c6da34304d0764fc7a4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/908c475345ba4daeb2c3934d36b6e5b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/53d78377a55d4a24b08811bd816b3a17, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/ebd508c871574bbcae622a789df6f2c4, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/261def4a64df4daabcd07272d7dcda59, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b] to archive 2024-12-10T00:17:07,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/9a397a6f14ac4c6da34304d0764fc7a4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/9a397a6f14ac4c6da34304d0764fc7a4 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4bf61c18d084b5e97b8e4c175fa8e6f 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/88766c9c284946cbbfafc5bb9340c989 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/cb50850106c948669fd10423f37518ae 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d9ed62149f8a48eda05c5e7fcd11b57a 2024-12-10T00:17:07,081 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/908c475345ba4daeb2c3934d36b6e5b6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/908c475345ba4daeb2c3934d36b6e5b6 2024-12-10T00:17:07,082 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/fd03b1b489264a03803c06b082fb7b40 2024-12-10T00:17:07,082 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/854177dddf77413ba4a572495b7f44fc 2024-12-10T00:17:07,082 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/53d78377a55d4a24b08811bd816b3a17 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/53d78377a55d4a24b08811bd816b3a17 2024-12-10T00:17:07,082 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/7a96a9a6a93c415ebc100e79194e1b1d 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/56b4f8f414d9412ab90f911c858eb7b6 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/54e111d9cfa04c87849b254edaf8bc1e 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/d4369ce874354604a65745594da82020 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/261def4a64df4daabcd07272d7dcda59 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/261def4a64df4daabcd07272d7dcda59 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/12106c68b088484fa0d4b7504cee392a 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/ebd508c871574bbcae622a789df6f2c4 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/ebd508c871574bbcae622a789df6f2c4 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/2606116184e74a52a4aa5debae31320a 2024-12-10T00:17:07,083 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/8769c3cf13d041beb5c058867b5a108f 2024-12-10T00:17:07,084 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/bf7f1df343e6432ebb9195e2d703c723 2024-12-10T00:17:07,084 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/98d9a1b6d1e24926ac57da58e8d7a16b 2024-12-10T00:17:07,086 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=4 2024-12-10T00:17:07,087 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. 2024-12-10T00:17:07,087 DEBUG [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 62785890ba50211da2fffffcad482daf: Waiting for close lock at 1733789825783Running coprocessor pre-close hooks at 1733789825783Disabling compacts and flushes for region at 1733789825783Disabling writes for close at 1733789825783Obtaining lock to block concurrent updates at 1733789825783Preparing flush snapshotting stores in 62785890ba50211da2fffffcad482daf at 1733789825783Finished memstore snapshotting TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733789825784 (+1 ms)Flushing stores of TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf. at 1733789825784Flushing 62785890ba50211da2fffffcad482daf/A: creating writer at 1733789825784Flushing mob file 62785890ba50211da2fffffcad482daf/A: appending metadata at 1733789825792 (+8 ms)Flushing mob file 62785890ba50211da2fffffcad482daf/A: closing flushed file at 1733789825792Flushing 62785890ba50211da2fffffcad482daf/A: appending metadata at 1733789826206 (+414 ms)Flushing 62785890ba50211da2fffffcad482daf/A: closing flushed file at 1733789826206Flushing 62785890ba50211da2fffffcad482daf/B: creating writer at 1733789826620 (+414 ms)Flushing 62785890ba50211da2fffffcad482daf/B: appending metadata at 1733789826622 (+2 ms)Flushing 62785890ba50211da2fffffcad482daf/B: closing flushed file at 1733789826622Flushing 62785890ba50211da2fffffcad482daf/C: creating writer at 1733789827037 (+415 ms)Flushing 62785890ba50211da2fffffcad482daf/C: appending metadata at 1733789827040 (+3 ms)Flushing 62785890ba50211da2fffffcad482daf/C: closing flushed file at 1733789827040Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@709acda5: reopening flushed file at 1733789827049 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c6a83cf: reopening flushed file at 1733789827054 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18a6026: reopening flushed file at 1733789827059 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 62785890ba50211da2fffffcad482daf in 1282ms, sequenceid=365, compaction requested=true at 1733789827065 (+6 ms)Writing region close event to WAL at 1733789827084 (+19 ms)Running coprocessor post-close hooks at 1733789827087 (+3 ms)Closed at 1733789827087 2024-12-10T00:17:07,088 INFO [RS_CLOSE_REGION-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,088 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=62785890ba50211da2fffffcad482daf, regionState=CLOSED 2024-12-10T00:17:07,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 because future has completed 2024-12-10T00:17:07,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-10T00:17:07,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 62785890ba50211da2fffffcad482daf, server=a67c4886b4f7,36911,1733789645811 in 1.4600 sec 2024-12-10T00:17:07,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-10T00:17:07,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62785890ba50211da2fffffcad482daf, UNASSIGN in 1.4640 sec 2024-12-10T00:17:07,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-10T00:17:07,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-12-10T00:17:07,094 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733789827094"}]},"ts":"1733789827094"} 2024-12-10T00:17:07,095 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T00:17:07,095 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T00:17:07,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4770 sec 2024-12-10T00:17:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-10T00:17:07,771 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:17:07,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-10T00:17:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-10T00:17:07,780 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,782 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,784 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits] 2024-12-10T00:17:07,789 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/66aace7b465f4b02bee790424148e2f0 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/66aace7b465f4b02bee790424148e2f0 2024-12-10T00:17:07,789 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/234ff83cab604596a43e278391b2706d to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/234ff83cab604596a43e278391b2706d 2024-12-10T00:17:07,789 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/b5025343934442acb0db2706d4bbb096 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/b5025343934442acb0db2706d4bbb096 2024-12-10T00:17:07,789 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e30dee084312439a85795217b78b2823 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/A/e30dee084312439a85795217b78b2823 2024-12-10T00:17:07,793 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/3d0f0a0806e8407bbb1722e4fe7fa032 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/3d0f0a0806e8407bbb1722e4fe7fa032 2024-12-10T00:17:07,793 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/abb3fc01d638461cab1b18211601f5e6 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/abb3fc01d638461cab1b18211601f5e6 2024-12-10T00:17:07,794 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ddd1282f9ef34ba99f2f4dcc6b648518 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/ddd1282f9ef34ba99f2f4dcc6b648518 2024-12-10T00:17:07,794 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/10b6732701b24193a31945bd04032104 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/B/10b6732701b24193a31945bd04032104 2024-12-10T00:17:07,799 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/007acb6390dc46e38200d8eed1051f9b to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/007acb6390dc46e38200d8eed1051f9b 2024-12-10T00:17:07,799 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a04badc64ca342d0bfce9aedfce65798 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a04badc64ca342d0bfce9aedfce65798 2024-12-10T00:17:07,799 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a2ebdfddd0b041bc985e020c3b0f2dcf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/a2ebdfddd0b041bc985e020c3b0f2dcf 2024-12-10T00:17:07,799 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/3de020e9c30247c8afb2461b5dbf9099 to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/C/3de020e9c30247c8afb2461b5dbf9099 2024-12-10T00:17:07,803 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits/368.seqid to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf/recovered.edits/368.seqid 2024-12-10T00:17:07,804 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/default/TestAcidGuarantees/62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T00:17:07,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:17:07,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T00:17:07,816 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210065808890dd54482aca47bbf4691fb47_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210065808890dd54482aca47bbf4691fb47_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,816 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100a49374adbf44ca0ab82c287fa72836d_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100a49374adbf44ca0ab82c287fa72836d_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016ddd1e7570d4c8099f197305e908f10_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121016ddd1e7570d4c8099f197305e908f10_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101495f9ec078340c3a182bfed01d98ab3_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101495f9ec078340c3a182bfed01d98ab3_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101dec37e89f0e4369ab28bc1476bd0ecd_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101dec37e89f0e4369ab28bc1476bd0ecd_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042681967a7274121ab750198d3318c05_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121042681967a7274121ab750198d3318c05_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d7c0696c88444118ed50c1626d1f286_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d7c0696c88444118ed50c1626d1f286_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,817 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121033641bdde1b042289eca153b0059b743_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121033641bdde1b042289eca153b0059b743_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e0985a53b734cec8716504359aa9243_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105e0985a53b734cec8716504359aa9243_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b72e22fbce94db4a3f52d56a5443788_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108b72e22fbce94db4a3f52d56a5443788_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108639052ec5f14150897858083e12e9c2_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108639052ec5f14150897858083e12e9c2_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108de05a9b784645aa9f8e029ed3bb0cec_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108de05a9b784645aa9f8e029ed3bb0cec_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093990f367a224ff8a681a344ed69c416_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093990f367a224ff8a681a344ed69c416_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,819 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093bdb407021e47fda88fec65c3f23306_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121093bdb407021e47fda88fec65c3f23306_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,820 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cab6fa4705244404bf04156277f872b0_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210cab6fa4705244404bf04156277f872b0_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,820 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1ed472bff154b4aa8bac08ddb59027d_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1ed472bff154b4aa8bac08ddb59027d_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,820 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27a5c6b10a34e3ead41fa374cfcddf1_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f27a5c6b10a34e3ead41fa374cfcddf1_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,820 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc4495307cab4e97a5d48f13a5494c6a_62785890ba50211da2fffffcad482daf to hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc4495307cab4e97a5d48f13a5494c6a_62785890ba50211da2fffffcad482daf 2024-12-10T00:17:07,821 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T00:17:07,822 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,824 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T00:17:07,826 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T00:17:07,827 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,827 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T00:17:07,827 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733789827827"}]},"ts":"9223372036854775807"} 2024-12-10T00:17:07,828 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T00:17:07,828 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 62785890ba50211da2fffffcad482daf, NAME => 'TestAcidGuarantees,,1733789793847.62785890ba50211da2fffffcad482daf.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T00:17:07,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T00:17:07,829 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733789827828"}]},"ts":"9223372036854775807"} 2024-12-10T00:17:07,830 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T00:17:07,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T00:17:07,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-12-10T00:17:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-10T00:17:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T00:17:07,890 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T00:17:07,898 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254 (was 257), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 503), ProcessCount=11 (was 11), AvailableMemoryMB=7050 (was 7062) 2024-12-10T00:17:07,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T00:17:07,898 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:17:07,898 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:17:07,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:07,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:07,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T00:17:07,899 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T00:17:07,899 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-12-10T00:17:07,899 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a67c4886b4f7,35723,1733789644834 2024-12-10T00:17:07,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:17:07,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T00:17:07,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:17:07,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:17:07,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:17:07,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T00:17:07,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:17:07,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:07,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:17:07,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T00:17:07,980 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a67c4886b4f7,36911,1733789645811' ***** 2024-12-10T00:17:07,980 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T00:17:07,981 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T00:17:07,981 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T00:17:07,982 INFO [RS:0;a67c4886b4f7:36911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T00:17:07,982 INFO [RS:0;a67c4886b4f7:36911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T00:17:07,982 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(959): stopping server a67c4886b4f7,36911,1733789645811 2024-12-10T00:17:07,982 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:17:07,982 INFO [RS:0;a67c4886b4f7:36911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a67c4886b4f7:36911. 2024-12-10T00:17:07,982 DEBUG [RS:0;a67c4886b4f7:36911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T00:17:07,982 DEBUG [RS:0;a67c4886b4f7:36911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:07,983 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T00:17:07,983 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T00:17:07,983 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T00:17:07,983 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T00:17:07,984 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T00:17:07,984 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T00:17:07,984 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:07,984 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T00:17:07,984 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T00:17:07,985 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T00:17:07,985 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T00:17:07,985 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T00:17:07,985 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-10T00:17:07,998 INFO [regionserver/a67c4886b4f7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:17:08,006 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/info/2480f18db21545509fde202d3feb02f4 is 95, key is TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806./info:/1733789677619/DeleteFamily/seqid=0 2024-12-10T00:17:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742455_1631 (size=6514) 2024-12-10T00:17:08,185 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:08,385 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:08,410 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/info/2480f18db21545509fde202d3feb02f4 2024-12-10T00:17:08,433 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/ns/f2f88f6896fc440980ec3500885e5426 is 93, key is TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806./ns:/1733789677569/DeleteFamily/seqid=0 2024-12-10T00:17:08,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742456_1632 (size=5906) 2024-12-10T00:17:08,585 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:08,630 INFO [regionserver/a67c4886b4f7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-12-10T00:17:08,630 INFO [regionserver/a67c4886b4f7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T00:17:08,665 INFO [regionserver/a67c4886b4f7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T00:17:08,786 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:08,857 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/ns/f2f88f6896fc440980ec3500885e5426 2024-12-10T00:17:08,885 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/rep_barrier/4ed2671ba7d247c9b1f9e838e2d79166 is 102, key is TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806./rep_barrier:/1733789677569/DeleteFamily/seqid=0 2024-12-10T00:17:08,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742457_1633 (size=6025) 2024-12-10T00:17:08,986 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:09,186 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:09,290 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/rep_barrier/4ed2671ba7d247c9b1f9e838e2d79166 2024-12-10T00:17:09,316 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/table/42a8b81dfa8f40f996c0585bd20a23bd is 96, key is TestAcidGuarantees,,1733789650041.a5f283ed44d3b86dba39726447c58806./table:/1733789677569/DeleteFamily/seqid=0 2024-12-10T00:17:09,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742458_1634 (size=5936) 2024-12-10T00:17:09,386 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:09,587 DEBUG [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T00:17:09,720 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/table/42a8b81dfa8f40f996c0585bd20a23bd 2024-12-10T00:17:09,730 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/info/2480f18db21545509fde202d3feb02f4 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/info/2480f18db21545509fde202d3feb02f4 2024-12-10T00:17:09,734 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/info/2480f18db21545509fde202d3feb02f4, entries=12, sequenceid=90, filesize=6.4 K 2024-12-10T00:17:09,735 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/ns/f2f88f6896fc440980ec3500885e5426 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/ns/f2f88f6896fc440980ec3500885e5426 2024-12-10T00:17:09,739 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/ns/f2f88f6896fc440980ec3500885e5426, entries=8, sequenceid=90, filesize=5.8 K 2024-12-10T00:17:09,740 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/rep_barrier/4ed2671ba7d247c9b1f9e838e2d79166 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/rep_barrier/4ed2671ba7d247c9b1f9e838e2d79166 2024-12-10T00:17:09,744 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/rep_barrier/4ed2671ba7d247c9b1f9e838e2d79166, entries=6, sequenceid=90, filesize=5.9 K 2024-12-10T00:17:09,745 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/.tmp/table/42a8b81dfa8f40f996c0585bd20a23bd as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/table/42a8b81dfa8f40f996c0585bd20a23bd 2024-12-10T00:17:09,747 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/table/42a8b81dfa8f40f996c0585bd20a23bd, entries=7, sequenceid=90, filesize=5.8 K 2024-12-10T00:17:09,748 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1763ms, sequenceid=90, compaction requested=false 2024-12-10T00:17:09,751 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-10T00:17:09,752 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T00:17:09,752 INFO [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T00:17:09,752 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733789827984Running coprocessor pre-close hooks at 1733789827984Disabling compacts and flushes for region at 1733789827984Disabling writes for close at 1733789827985 (+1 ms)Obtaining lock to block concurrent updates at 1733789827985Preparing flush snapshotting stores in 1588230740 at 1733789827985Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733789827986 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733789827986Flushing 1588230740/info: creating writer at 1733789827987 (+1 ms)Flushing 1588230740/info: appending metadata at 1733789828006 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733789828006Flushing 1588230740/ns: creating writer at 1733789828415 (+409 ms)Flushing 1588230740/ns: appending metadata at 1733789828433 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733789828433Flushing 1588230740/rep_barrier: creating writer at 1733789828867 (+434 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733789828885 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733789828885Flushing 1588230740/table: creating writer at 1733789829300 (+415 ms)Flushing 1588230740/table: appending metadata at 1733789829315 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733789829315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e5dde57: reopening flushed file at 1733789829729 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f0a4b5: reopening flushed file at 1733789829735 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49877b2c: reopening flushed file at 1733789829739 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dadf7d7: reopening flushed file at 1733789829744 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1763ms, sequenceid=90, compaction requested=false at 1733789829748 (+4 ms)Writing region close event to WAL at 1733789829749 (+1 ms)Running coprocessor post-close hooks at 1733789829752 (+3 ms)Closed at 1733789829752 2024-12-10T00:17:09,752 DEBUG [RS_CLOSE_META-regionserver/a67c4886b4f7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T00:17:09,787 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(976): stopping server a67c4886b4f7,36911,1733789645811; all regions closed. 2024-12-10T00:17:09,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741834_1010 (size=25344) 2024-12-10T00:17:09,794 DEBUG [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/oldWALs 2024-12-10T00:17:09,794 INFO [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67c4886b4f7%2C36911%2C1733789645811.meta:.meta(num 1733789649317) 2024-12-10T00:17:09,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741833_1009 (size=18890956) 2024-12-10T00:17:09,799 DEBUG [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/oldWALs 2024-12-10T00:17:09,799 INFO [RS:0;a67c4886b4f7:36911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67c4886b4f7%2C36911%2C1733789645811:(num 1733789648809) 2024-12-10T00:17:09,799 DEBUG [RS:0;a67c4886b4f7:36911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T00:17:09,799 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T00:17:09,799 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:17:09,799 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.ChoreService(370): Chore service for: regionserver/a67c4886b4f7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T00:17:09,800 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:17:09,800 INFO [regionserver/a67c4886b4f7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:17:09,801 INFO [RS:0;a67c4886b4f7:36911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36911 2024-12-10T00:17:09,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T00:17:09,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a67c4886b4f7,36911,1733789645811 2024-12-10T00:17:09,852 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:17:09,854 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a67c4886b4f7,36911,1733789645811] 2024-12-10T00:17:09,878 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a67c4886b4f7,36911,1733789645811 already deleted, retry=false 2024-12-10T00:17:09,878 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a67c4886b4f7,36911,1733789645811 expired; onlineServers=0 2024-12-10T00:17:09,879 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a67c4886b4f7,35723,1733789644834' ***** 2024-12-10T00:17:09,879 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T00:17:09,879 INFO [M:0;a67c4886b4f7:35723 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T00:17:09,879 INFO [M:0;a67c4886b4f7:35723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T00:17:09,880 DEBUG [M:0;a67c4886b4f7:35723 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T00:17:09,880 DEBUG [M:0;a67c4886b4f7:35723 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T00:17:09,880 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T00:17:09,880 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.small.0-1733789648400 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.small.0-1733789648400,5,FailOnTimeoutGroup] 2024-12-10T00:17:09,880 DEBUG [master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.large.0-1733789648399 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67c4886b4f7:0:becomeActiveMaster-HFileCleaner.large.0-1733789648399,5,FailOnTimeoutGroup] 2024-12-10T00:17:09,881 INFO [M:0;a67c4886b4f7:35723 {}] hbase.ChoreService(370): Chore service for: master/a67c4886b4f7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T00:17:09,881 INFO [M:0;a67c4886b4f7:35723 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T00:17:09,881 DEBUG [M:0;a67c4886b4f7:35723 {}] master.HMaster(1795): Stopping service threads 2024-12-10T00:17:09,881 INFO [M:0;a67c4886b4f7:35723 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T00:17:09,881 INFO [M:0;a67c4886b4f7:35723 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T00:17:09,882 INFO [M:0;a67c4886b4f7:35723 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T00:17:09,882 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T00:17:09,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T00:17:09,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T00:17:09,894 DEBUG [M:0;a67c4886b4f7:35723 {}] zookeeper.ZKUtil(347): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T00:17:09,894 WARN [M:0;a67c4886b4f7:35723 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T00:17:09,895 INFO [M:0;a67c4886b4f7:35723 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/.lastflushedseqids 2024-12-10T00:17:09,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742459_1635 (size=72) 2024-12-10T00:17:09,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:17:09,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1000d127b280001, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:17:09,969 INFO [RS:0;a67c4886b4f7:36911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:17:09,969 INFO [RS:0;a67c4886b4f7:36911 {}] regionserver.HRegionServer(1031): Exiting; stopping=a67c4886b4f7,36911,1733789645811; zookeeper connection closed. 2024-12-10T00:17:09,970 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3da18a94 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3da18a94 2024-12-10T00:17:09,970 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T00:17:10,309 INFO [M:0;a67c4886b4f7:35723 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T00:17:10,309 INFO [M:0;a67c4886b4f7:35723 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T00:17:10,310 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T00:17:10,310 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:17:10,310 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:17:10,310 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T00:17:10,310 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:17:10,310 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=772.96 KB heapSize=946.30 KB 2024-12-10T00:17:10,331 DEBUG [M:0;a67c4886b4f7:35723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/35026a094769401380ac4cc995ff507e is 82, key is hbase:meta,,1/info:regioninfo/1733789649444/Put/seqid=0 2024-12-10T00:17:10,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742460_1636 (size=5672) 2024-12-10T00:17:10,735 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/35026a094769401380ac4cc995ff507e 2024-12-10T00:17:10,759 DEBUG [M:0;a67c4886b4f7:35723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2ade2925d9c4ef5a228f3580f319735 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1733789797131/Put/seqid=0 2024-12-10T00:17:10,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742461_1637 (size=45773) 2024-12-10T00:17:11,163 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=772.41 KB at sequenceid=2137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2ade2925d9c4ef5a228f3580f319735 2024-12-10T00:17:11,171 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a2ade2925d9c4ef5a228f3580f319735 2024-12-10T00:17:11,189 DEBUG [M:0;a67c4886b4f7:35723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a037cf8af904fb28662d694ecd09d3a is 69, key is a67c4886b4f7,36911,1733789645811/rs:state/1733789648511/Put/seqid=0 2024-12-10T00:17:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073742462_1638 (size=5156) 2024-12-10T00:17:11,592 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a037cf8af904fb28662d694ecd09d3a 2024-12-10T00:17:11,596 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/35026a094769401380ac4cc995ff507e as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/35026a094769401380ac4cc995ff507e 2024-12-10T00:17:11,599 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/35026a094769401380ac4cc995ff507e, entries=8, sequenceid=2137, filesize=5.5 K 2024-12-10T00:17:11,600 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a2ade2925d9c4ef5a228f3580f319735 as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a2ade2925d9c4ef5a228f3580f319735 2024-12-10T00:17:11,603 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a2ade2925d9c4ef5a228f3580f319735 2024-12-10T00:17:11,603 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a2ade2925d9c4ef5a228f3580f319735, entries=172, sequenceid=2137, filesize=44.7 K 2024-12-10T00:17:11,603 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a037cf8af904fb28662d694ecd09d3a as hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a037cf8af904fb28662d694ecd09d3a 2024-12-10T00:17:11,606 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a037cf8af904fb28662d694ecd09d3a, entries=1, sequenceid=2137, filesize=5.0 K 2024-12-10T00:17:11,607 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(3140): Finished flush of dataSize ~772.96 KB/791512, heapSize ~946 KB/968704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2137, compaction requested=false 2024-12-10T00:17:11,608 INFO [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T00:17:11,608 DEBUG [M:0;a67c4886b4f7:35723 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733789830310Disabling compacts and flushes for region at 1733789830310Disabling writes for close at 1733789830310Obtaining lock to block concurrent updates at 1733789830310Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733789830310Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=791512, getHeapSize=968944, getOffHeapSize=0, getCellsCount=2334 at 1733789830311 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733789830312 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733789830313 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733789830331 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733789830331Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733789830741 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733789830759 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733789830759Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733789831171 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733789831189 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733789831189Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57709d3e: reopening flushed file at 1733789831595 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@737f54db: reopening flushed file at 1733789831599 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6adb67bf: reopening flushed file at 1733789831603 (+4 ms)Finished flush of dataSize ~772.96 KB/791512, heapSize ~946 KB/968704, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2137, compaction requested=false at 1733789831607 (+4 ms)Writing region close event to WAL at 1733789831608 (+1 ms)Closed at 1733789831608 2024-12-10T00:17:11,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33323 is added to blk_1073741830_1006 (size=931624) 2024-12-10T00:17:11,611 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c1405419-ade9-42fc-45b0-d65b453b5410/MasterData/WALs/a67c4886b4f7,35723,1733789644834/a67c4886b4f7%2C35723%2C1733789644834.1733789647420 not finished, retry = 0 2024-12-10T00:17:11,712 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T00:17:11,713 INFO [M:0;a67c4886b4f7:35723 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T00:17:11,713 INFO [M:0;a67c4886b4f7:35723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35723 2024-12-10T00:17:11,713 INFO [M:0;a67c4886b4f7:35723 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T00:17:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:17:11,868 INFO [M:0;a67c4886b4f7:35723 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T00:17:11,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35723-0x1000d127b280000, quorum=127.0.0.1:50532, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T00:17:11,879 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T00:17:11,882 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T00:17:11,882 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T00:17:11,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T00:17:11,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.log.dir/,STOPPED} 2024-12-10T00:17:11,884 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T00:17:11,884 WARN [BP-85213971-172.17.0.2-1733789639131 heartbeating to localhost/127.0.0.1:37473 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T00:17:11,884 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T00:17:11,884 WARN [BP-85213971-172.17.0.2-1733789639131 heartbeating to localhost/127.0.0.1:37473 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-85213971-172.17.0.2-1733789639131 (Datanode Uuid ede03299-6f89-48ab-bc21-9d22c2a3de54) service to localhost/127.0.0.1:37473 2024-12-10T00:17:11,886 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/data/data1/current/BP-85213971-172.17.0.2-1733789639131 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T00:17:11,887 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/cluster_af7da76e-2c20-78ff-f53f-00d159bfba5b/data/data2/current/BP-85213971-172.17.0.2-1733789639131 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T00:17:11,887 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T00:17:11,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T00:17:11,893 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T00:17:11,893 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T00:17:11,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T00:17:11,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce45382b-cae6-3822-df06-e001f3b1d072/hadoop.log.dir/,STOPPED} 2024-12-10T00:17:11,908 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T00:17:12,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down